mirror of
https://github.com/projectsend/projectsend.git
synced 2026-10-05 21:01:33 +00:00
veenone
4b30849a88
The header checkbox on Import orphan files selected only the 25 rows on screen, so an install with thousands of stray files had to import them a page at a time. Once a whole page is ticked, the selection bar now offers "Select all N matching files", and "Import all" takes every orphan the search matches, on every page. The import runs in a queued job because it is too slow for a request. Each file is hashed in full and written in three commits, so 5,000 files of 4 MB take about four minutes, and PHP stops a request after 30 s of CPU, around file 1,100. ImportOrphanFilesJob works on the default queue in chunks of about 45 s: each chunk rescans, imports what is still orphaned and queues the next one. That keeps every job inside the worker's 60 s timeout and the queue's 90 s retry_after, so no extra worker is needed, and mail queued in the meantime goes out between chunks. If a run dies part way, the next one picks up what is left. Only one run can be active at a time. OrphanImportProgress keeps its state in the cache and starts a run under a lock. While a run is active, every other import is refused, the per-row button included, so no file is adopted twice. The page polls files/orphans/import-status every 3 s and shows the run as running, finished, failed with the reason, or stalled after 5 minutes without progress, which usually means no worker is listening. Bulk delete still works one page at a time. The adoption itself moved to OrphanFileImporter so the request and the job share it, and the rule for what can be imported now lives in OrphanFileScanner::importable().
80 lines
2.6 KiB
PHP
80 lines
2.6 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Modules\Files\Jobs;
|
|
|
|
use App\Models\User;
|
|
use App\Modules\Files\OrphanFileImporter;
|
|
use App\Modules\Files\OrphanFileScanner;
|
|
use App\Modules\Files\OrphanImportProgress;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Throwable;
|
|
|
|
/**
|
|
* "Import all" on the orphans screen. Thousands of files, each hashed in
|
|
* full, run for minutes, which is far past what one web request is
|
|
* allowed (see OrphanFilesController::import).
|
|
*
|
|
* The work is cut into chunks of about $budgetSeconds. Each chunk scans
|
|
* the disk again and imports what is still orphaned until its time is up,
|
|
* then queues the next chunk. That keeps every job well inside the default
|
|
* worker's 60s timeout and the queue's 90s retry_after, so it can share
|
|
* the default queue (no extra worker to deploy). Mail queued in the
|
|
* meantime goes out between chunks rather than waiting behind the whole
|
|
* import. Because each chunk rescans, a file already adopted is never
|
|
* offered twice, and a run that dies part-way resumes from what is left.
|
|
*/
|
|
class ImportOrphanFilesJob implements ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable;
|
|
|
|
/**
|
|
* Not retried as such: the next "Import all" (or the next chunk)
|
|
* rescans and carries on from where this one stopped. failed()
|
|
* records why it stopped.
|
|
*/
|
|
public int $tries = 1;
|
|
|
|
public function __construct(
|
|
private readonly int $userId,
|
|
private readonly ?string $search,
|
|
private readonly int $budgetSeconds = 45,
|
|
) {}
|
|
|
|
public function handle(OrphanFileScanner $scanner, OrphanFileImporter $importer, OrphanImportProgress $progress): void
|
|
{
|
|
$user = User::query()->find($this->userId);
|
|
|
|
if ($user === null) {
|
|
$progress->fail('The account that started the import no longer exists.');
|
|
|
|
return;
|
|
}
|
|
|
|
$deadline = microtime(true) + $this->budgetSeconds;
|
|
|
|
foreach ($scanner->importable($user, $this->search) as $i => $item) {
|
|
// At least one file per chunk, so a run always moves forward.
|
|
if ($i > 0 && microtime(true) >= $deadline) {
|
|
self::dispatch($this->userId, $this->search, $this->budgetSeconds);
|
|
|
|
return;
|
|
}
|
|
|
|
$importer->import($user, $item['disk'], $item['path']);
|
|
$progress->advance();
|
|
}
|
|
|
|
$progress->finish();
|
|
}
|
|
|
|
public function failed(Throwable $exception): void
|
|
{
|
|
app(OrphanImportProgress::class)->fail($exception->getMessage());
|
|
}
|
|
}
|