Let "Import all" adopt every orphan the search matches, in a background job

The header checkbox on Import orphan files selected only the 25 rows on
screen, so an install with thousands of stray files had to import them a
page at a time. Once a whole page is ticked, the selection bar now offers
"Select all N matching files", and "Import all" takes every orphan the
search matches, on every page.

The import runs in a queued job because it is too slow for a request.
Each file is hashed in full and written in three commits, so 5,000 files
of 4 MB take about four minutes, and PHP stops a request after 30 s of
CPU, around file 1,100. ImportOrphanFilesJob works on the default queue in
chunks of about 45 s: each chunk rescans, imports what is still orphaned
and queues the next one. That keeps every job inside the worker's 60 s
timeout and the queue's 90 s retry_after, so no extra worker is needed,
and mail queued in the meantime goes out between chunks. If a run dies
part way, the next one picks up what is left.

Only one run can be active at a time. OrphanImportProgress keeps its state
in the cache and starts a run under a lock. While a run is active, every
other import is refused, the per-row button included, so no file is
adopted twice. The page polls files/orphans/import-status every 3 s and
shows the run as running, finished, failed with the reason, or stalled
after 5 minutes without progress, which usually means no worker is
listening.

Bulk delete still works one page at a time. The adoption itself moved to
OrphanFileImporter so the request and the job share it, and the rule for
what can be imported now lives in OrphanFileScanner::importable().

app/Modules/Files/Http/Controllers/OrphanFilesController.php

@@ -5,20 +5,24 @@ declare(strict_types=1);
 namespace App\Modules\Files\Http\Controllers;
 
 use App\Http\Controllers\Controller;
+use App\Models\User;
 use App\Modules\Audit\Action;
 use App\Modules\Audit\ActivityLogger;
+use App\Modules\Files\Jobs\ImportOrphanFilesJob;
 use App\Modules\Files\Models\File;
+use App\Modules\Files\OrphanFileImporter;
 use App\Modules\Files\OrphanFileScanner;
+use App\Modules\Files\OrphanImportProgress;
 use App\Modules\Files\Scanning\ScanStatus;
-use App\Modules\Files\Uploads\StoreUploadedFile;
 use App\Support\Pagination;
-use Illuminate\Contracts\Filesystem\Filesystem;
+use Illuminate\Http\JsonResponse;
 use Illuminate\Http\RedirectResponse;
 use Illuminate\Http\Request;
 use Illuminate\Pagination\LengthAwarePaginator;
 use Illuminate\Pagination\Paginator;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Validation\Rule;
+use Illuminate\Validation\ValidationException;
 use Inertia\Inertia;
 use Inertia\Response;
 
@@ -33,7 +37,8 @@ class OrphanFilesController extends Controller
 {
     public function __construct(
         private readonly OrphanFileScanner $scanner,
-        private readonly StoreUploadedFile $storeFile,
+        private readonly OrphanFileImporter $importer,
+        private readonly OrphanImportProgress $progress,
         private readonly ActivityLogger $activity,
     ) {}
 
@@ -91,6 +96,7 @@ class OrphanFilesController extends Controller
             'search' => $search,
             'scanned_disks' => $this->scanner->scannedDisks(),
             'missing_count' => File::query()->where('scan_status', ScanStatus::Missing)->count(),
+            'import_run' => $this->progress->current(),
         ]);
     }
 
@@ -138,14 +144,20 @@ class OrphanFilesController extends Controller
         $user = $request->user();
         assert($user !== null);
 
-        $validated = $this->validateItems($request);
+        if ($request->boolean('all')) {
+            return $this->importAll($request, $user);
+        }
+
+        // While a background run is adopting files, a second importer
+        // could adopt the same path twice.
+        if ($this->progress->isActive()) {
+            $this->alreadyRunning();
+        }
 
         $imported = 0;
         $importedFile = null;
 
-        foreach ($validated['items'] as $item) {
-            $disk = Storage::disk($item['disk']);
-
+        foreach ($this->validateItems($request)['items'] as $item) {
             // Re-validate against a fresh scan — never trust a
             // client-supplied disk/path just because an earlier scan
             // listed it.
@@ -153,18 +165,7 @@ class OrphanFilesController extends Controller
                 continue;
             }
 
-            $importedFile = $this->storeFile->create(
-                uploader: $user,
-                originalName: basename($item['path']),
-                path: $item['path'],
-                mimeType: $disk->mimeType($item['path']) ?: 'application/octet-stream',
-                size: $disk->size($item['path']),
-                checksum: $this->checksumOf($disk, $item['path']),
-                folderId: null,
-                disk: $item['disk'],
-                action: Action::FileImported,
-            );
-
+            $importedFile = $this->importer->import($user, $item['disk'], $item['path']);
             $imported++;
         }
 
@@ -182,6 +183,49 @@ class OrphanFilesController extends Controller
         ));
     }
 
+    /**
+     * Every orphan the search matches, on every page. Handed to a queued
+     * job: thousands of files hashed in full take minutes, and a request
+     * is cut off after 30s of CPU — part-way through, with an error page
+     * for an import that was in fact half done. See ImportOrphanFilesJob.
+     */
+    private function importAll(Request $request, User $user): RedirectResponse
+    {
+        $search = trim($request->validate(['search' => ['nullable', 'string', 'max:255']])['search'] ?? '');
+        $search = $search !== '' ? $search : null;
+
+        $total = count($this->scanner->importable($user, $search));
+
+        if ($total === 0) {
+            return back()->with('success', trans_choice(':count file imported.|:count files imported.', 0, ['count' => '0']));
+        }
+
+        if (! $this->progress->tryStart($total)) {
+            $this->alreadyRunning();
+        }
+
+        ImportOrphanFilesJob::dispatch($user->id, $search);
+
+        return back()->with('success', trans_choice(
+            'Importing :count file in the background.|Importing :count files in the background.',
+            $total,
+            ['count' => (string) $total],
+        ));
+    }
+
+    private function alreadyRunning(): never
+    {
+        throw ValidationException::withMessages(['items' => __('An import is already running. Wait for it to finish.')]);
+    }
+
+    /**
+     * Polled by the orphans screen while a background run is going.
+     */
+    public function importStatus(): JsonResponse
+    {
+        return response()->json($this->progress->current());
+    }
+
     public function destroy(Request $request): RedirectResponse
     {
         $validated = $this->validateItems($request);
@@ -218,24 +262,4 @@ class OrphanFilesController extends Controller
             'items.*.path' => ['required', 'string'],
         ]);
     }
-
-    /**
-     * Streamed rather than hash_file() on a local path — the only way to
-     * checksum a file that might live on a non-local disk (S3 has no
-     * local filesystem path to hand hash_file()).
-     */
-    private function checksumOf(Filesystem $disk, string $path): string
-    {
-        $stream = $disk->readStream($path);
-
-        if ($stream === null) {
-            return '';
-        }
-
-        $context = hash_init('sha256');
-        hash_update_stream($context, $stream);
-        fclose($stream);
-
-        return hash_final($context);
-    }
 }

app/Modules/Files/Jobs/ImportOrphanFilesJob.php

@@ -0,0 +1,79 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Modules\Files\Jobs;
+
+use App\Models\User;
+use App\Modules\Files\OrphanFileImporter;
+use App\Modules\Files\OrphanFileScanner;
+use App\Modules\Files\OrphanImportProgress;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Throwable;
+
+/**
+ * "Import all" on the orphans screen. Thousands of files, each hashed in
+ * full, run for minutes, which is far past what one web request is
+ * allowed (see OrphanFilesController::import).
+ *
+ * The work is cut into chunks of about $budgetSeconds. Each chunk scans
+ * the disk again and imports what is still orphaned until its time is up,
+ * then queues the next chunk. That keeps every job well inside the default
+ * worker's 60s timeout and the queue's 90s retry_after, so it can share
+ * the default queue (no extra worker to deploy). Mail queued in the
+ * meantime goes out between chunks rather than waiting behind the whole
+ * import. Because each chunk rescans, a file already adopted is never
+ * offered twice, and a run that dies part-way resumes from what is left.
+ */
+class ImportOrphanFilesJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable;
+
+    /**
+     * Not retried as such: the next "Import all" (or the next chunk)
+     * rescans and carries on from where this one stopped. failed()
+     * records why it stopped.
+     */
+    public int $tries = 1;
+
+    public function __construct(
+        private readonly int $userId,
+        private readonly ?string $search,
+        private readonly int $budgetSeconds = 45,
+    ) {}
+
+    public function handle(OrphanFileScanner $scanner, OrphanFileImporter $importer, OrphanImportProgress $progress): void
+    {
+        $user = User::query()->find($this->userId);
+
+        if ($user === null) {
+            $progress->fail('The account that started the import no longer exists.');
+
+            return;
+        }
+
+        $deadline = microtime(true) + $this->budgetSeconds;
+
+        foreach ($scanner->importable($user, $this->search) as $i => $item) {
+            // At least one file per chunk, so a run always moves forward.
+            if ($i > 0 && microtime(true) >= $deadline) {
+                self::dispatch($this->userId, $this->search, $this->budgetSeconds);
+
+                return;
+            }
+
+            $importer->import($user, $item['disk'], $item['path']);
+            $progress->advance();
+        }
+
+        $progress->finish();
+    }
+
+    public function failed(Throwable $exception): void
+    {
+        app(OrphanImportProgress::class)->fail($exception->getMessage());
+    }
+}

app/Modules/Files/OrphanFileImporter.php

@@ -0,0 +1,62 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Modules\Files;
+
+use App\Models\User;
+use App\Modules\Audit\Action;
+use App\Modules\Files\Models\File;
+use App\Modules\Files\Uploads\StoreUploadedFile;
+use Illuminate\Contracts\Filesystem\Filesystem;
+use Illuminate\Support\Facades\Storage;
+
+/**
+ * Adopts an orphan in place: a File row for bytes already on disk. Shared
+ * by the orphans screen (a ticked selection, in the request) and
+ * ImportOrphanFilesJob ("import all", in the background). Which paths are
+ * importable is OrphanFileScanner's call.
+ */
+class OrphanFileImporter
+{
+    public function __construct(
+        private readonly StoreUploadedFile $storeFile,
+    ) {}
+
+    public function import(User $user, string $diskName, string $path): File
+    {
+        $disk = Storage::disk($diskName);
+
+        return $this->storeFile->create(
+            uploader: $user,
+            originalName: basename($path),
+            path: $path,
+            mimeType: $disk->mimeType($path) ?: 'application/octet-stream',
+            size: $disk->size($path),
+            checksum: $this->checksumOf($disk, $path),
+            folderId: null,
+            disk: $diskName,
+            action: Action::FileImported,
+        );
+    }
+
+    /**
+     * Streamed rather than hash_file() on a local path — the only way to
+     * checksum a file that might live on a non-local disk (S3 has no
+     * local filesystem path to hand hash_file()).
+     */
+    private function checksumOf(Filesystem $disk, string $path): string
+    {
+        $stream = $disk->readStream($path);
+
+        if ($stream === null) {
+            return '';
+        }
+
+        $context = hash_init('sha256');
+        hash_update_stream($context, $stream);
+        fclose($stream);
+
+        return hash_final($context);
+    }
+}

app/Modules/Files/OrphanFileScanner.php

@@ -100,6 +100,26 @@ class OrphanFileScanner
         return $orphans;
     }
 
+    /**
+     * Every orphan the search matches that $viewer may import: the same
+     * rule as isImportable(), applied to a whole scan. Taken from a fresh
+     * scan, so unlike a client-supplied list it needs no per-path re-check.
+     *
+     * @return list<array{disk: string, path: string}>
+     */
+    public function importable(User $viewer, ?string $search = null): array
+    {
+        $importable = [];
+
+        foreach ($this->scan($viewer, $search) as $orphan) {
+            if ($orphan['allowed'] && $orphan['size'] > 0) {
+                $importable[] = ['disk' => $orphan['disk'], 'path' => $orphan['path']];
+            }
+        }
+
+        return $importable;
+    }
+
     /**
      * Re-validated at import/delete time — never trust a client-supplied
      * disk/path just because it was in an earlier scan response. $disk

app/Modules/Files/OrphanImportProgress.php

@@ -0,0 +1,131 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Modules\Files;
+
+use Illuminate\Support\Facades\Cache;
+
+/**
+ * The one background "import all" run on the orphans screen: how far it
+ * has got, and whether it is still going. Kept in the cache rather than a
+ * table: it is a progress readout for whoever is watching, not a record
+ * (every imported file is already in the activity log).
+ *
+ * There is only ever one run. Two at once would each scan the same disk
+ * and adopt the same paths twice, so a new one is refused while the
+ * current one is active.
+ *
+ * @phpstan-type Run array{status: string, total: int, imported: int, error: ?string, started_at: int, updated_at: int}
+ */
+class OrphanImportProgress
+{
+    private const KEY = 'orphan-files:import-run';
+
+    /** A run kept for a day, so its outcome is still there after lunch. */
+    private const TTL_SECONDS = 86400;
+
+    /**
+     * Each chunk of the job runs for under a minute, and every imported
+     * file touches updated_at. A run silent for this long has no worker
+     * behind it (crashed, or no worker listening), and must not block
+     * the next one forever.
+     */
+    private const STALL_SECONDS = 300;
+
+    /**
+     * Starts a run unless one is active, under a lock so two people
+     * clicking "Import all" at the same moment cannot both get one.
+     */
+    public function tryStart(int $total): bool
+    {
+        return (bool) Cache::lock(self::KEY.':start', 10)->get(function () use ($total): bool {
+            if ($this->isActive()) {
+                return false;
+            }
+
+            $this->put([
+                'status' => 'running',
+                'total' => $total,
+                'imported' => 0,
+                'error' => null,
+                'started_at' => now()->getTimestamp(),
+            ]);
+
+            return true;
+        });
+    }
+
+    public function advance(): void
+    {
+        $run = $this->raw();
+
+        if ($run !== null) {
+            $run['imported']++;
+            $this->put($run);
+        }
+    }
+
+    public function finish(): void
+    {
+        $this->end('finished');
+    }
+
+    public function fail(string $error): void
+    {
+        $this->end('failed', $error);
+    }
+
+    public function isActive(): bool
+    {
+        return ($this->current()['status'] ?? null) === 'running';
+    }
+
+    /**
+     * The run as the page shows it. A running run that has gone quiet is
+     * reported as stalled.
+     *
+     * @return array{status: string, total: int, imported: int, error: ?string, started_at: int}|null
+     */
+    public function current(): ?array
+    {
+        $run = $this->raw();
+
+        if ($run === null) {
+            return null;
+        }
+
+        if ($run['status'] === 'running' && now()->getTimestamp() - $run['updated_at'] > self::STALL_SECONDS) {
+            $run['status'] = 'stalled';
+        }
+
+        unset($run['updated_at']);
+
+        return $run;
+    }
+
+    private function end(string $status, ?string $error = null): void
+    {
+        $run = $this->raw();
+
+        if ($run !== null) {
+            $this->put([...$run, 'status' => $status, 'error' => $error]);
+        }
+    }
+
+    /**
+     * @return Run|null
+     */
+    private function raw(): ?array
+    {
+        return Cache::get(self::KEY);
+    }
+
+    /**
+     * @param  array<string, mixed>  $run
+     */
+    private function put(array $run): void
+    {
+        Cache::put(self::KEY, [...$run, 'updated_at' => now()->getTimestamp()], self::TTL_SECONDS);
+    }
+}

routes/web.php

@@ -126,6 +126,7 @@ Route::middleware(['auth'])->group(function () {
     Route::middleware(['staff', 'can:import_orphans'])->group(function () {
         Route::get('files/orphans', [OrphanFilesController::class, 'index'])->name('orphan-files.index');
         Route::post('files/orphans/import', [OrphanFilesController::class, 'import'])->name('orphan-files.import');
+        Route::get('files/orphans/import-status', [OrphanFilesController::class, 'importStatus'])->name('orphan-files.import-status');
         Route::post('files/orphans/delete', [OrphanFilesController::class, 'destroy'])->name('orphan-files.delete');
     });
 

tests/Feature/Files/OrphanFilesTest.php

@@ -5,9 +5,12 @@ declare(strict_types=1);
 use App\Models\User;
 use App\Modules\Audit\Action;
 use App\Modules\Audit\ActivityLog;
+use App\Modules\Files\Jobs\ImportOrphanFilesJob;
 use App\Modules\Files\Models\File;
+use App\Modules\Files\OrphanImportProgress;
 use App\Modules\Platform\Settings\Setting;
 use App\Modules\Platform\Settings\Settings;
+use Illuminate\Support\Facades\Queue;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Testing\TestResponse;
 use Inertia\Testing\AssertableInertia;
@@ -31,6 +34,23 @@ function makeAdoptedFile(User $uploader, string $path, string $disk = 'files'):
     ]);
 }
 
+/**
+ * 30 importable .txt files under 2026/07/batch, plus one empty and one
+ * restricted file there and one importable file outside it.
+ */
+function makeBatchOrphans(): void
+{
+    app(Settings::class)->set(Setting::UploadTypeRestriction, 'all');
+    app(Settings::class)->set(Setting::AllowedUploadExtensions, ['txt']);
+
+    foreach (range(1, 30) as $i) {
+        makeOrphanFile(sprintf('2026/07/batch/%02d.txt', $i));
+    }
+    makeOrphanFile('2026/07/batch/empty.txt', '');
+    makeOrphanFile('2026/07/batch/shell.php');
+    makeOrphanFile('2026/07/elsewhere.txt');
+}
+
 function orphanImport(array $items): TestResponse
 {
     return test()->postJson('/files/orphans/import', ['items' => $items]);
@@ -215,6 +235,86 @@ test('importing multiple orphans at once stays on the list rather than picking o
     }
 });
 
+test('import all queues a background run for every importable match instead of importing in the request', function () {
+    Queue::fake();
+    makeBatchOrphans();
+
+    $this->actingAs($this->admin)
+        ->postJson('/files/orphans/import', ['all' => true, 'search' => 'batch'])
+        ->assertRedirect();
+
+    expect(File::query()->count())->toBe(0);
+    Queue::assertPushed(ImportOrphanFilesJob::class, 1);
+
+    $this->getJson('/files/orphans/import-status')
+        ->assertOk()
+        ->assertJson(['status' => 'running', 'total' => 30, 'imported' => 0]);
+});
+
+test('the background run imports every match across as many chunks as it takes, then reports finished', function () {
+    makeBatchOrphans();
+
+    app(OrphanImportProgress::class)->tryStart(30);
+
+    // A zero budget makes every chunk stop after one file, so the run has
+    // to hand over to the next chunk 29 times to finish.
+    ImportOrphanFilesJob::dispatch($this->admin->id, 'batch', budgetSeconds: 0);
+
+    expect(File::query()->count())->toBe(30)
+        ->and(File::query()->where('path', '2026/07/elsewhere.txt')->exists())->toBeFalse()
+        ->and(File::query()->where('uploaded_by', $this->admin->id)->count())->toBe(30);
+
+    $this->actingAs($this->admin)->getJson('/files/orphans/import-status')
+        ->assertJson(['status' => 'finished', 'total' => 30, 'imported' => 30]);
+});
+
+test('while a run is in progress no other import is accepted, so nothing is adopted twice', function () {
+    Queue::fake();
+    makeOrphanFile('2026/07/one.txt');
+
+    app(OrphanImportProgress::class)->tryStart(1);
+
+    $this->actingAs($this->admin);
+    orphanImport([['disk' => 'files', 'path' => '2026/07/one.txt']])->assertUnprocessable();
+    $this->postJson('/files/orphans/import', ['all' => true])->assertUnprocessable();
+
+    expect(File::query()->count())->toBe(0);
+    Queue::assertNothingPushed();
+});
+
+test('a run that stops making progress is reported as stalled and no longer blocks a new one', function () {
+    Queue::fake();
+    makeOrphanFile('2026/07/one.txt');
+
+    app(OrphanImportProgress::class)->tryStart(5);
+    $this->travel(6)->minutes();
+
+    $this->actingAs($this->admin)->getJson('/files/orphans/import-status')
+        ->assertJson(['status' => 'stalled', 'total' => 5, 'imported' => 0]);
+
+    $this->postJson('/files/orphans/import', ['all' => true])->assertRedirect();
+    Queue::assertPushed(ImportOrphanFilesJob::class, 1);
+});
+
+test('a run whose job fails is reported as failed with the reason', function () {
+    app(OrphanImportProgress::class)->tryStart(5);
+
+    (new ImportOrphanFilesJob($this->admin->id, null))->failed(new RuntimeException('Disk unreachable'));
+
+    $this->actingAs($this->admin)->getJson('/files/orphans/import-status')
+        ->assertJson(['status' => 'failed', 'error' => 'Disk unreachable']);
+});
+
+test('import all with nothing importable queues nothing', function () {
+    Queue::fake();
+    makeOrphanFile('2026/07/empty.txt', '');
+
+    $this->actingAs($this->admin)->postJson('/files/orphans/import', ['all' => true])->assertRedirect();
+
+    Queue::assertNothingPushed();
+    $this->getJson('/files/orphans/import-status')->assertOk()->assertExactJson([]);
+});
+
 test('a 0-byte orphan cannot be imported but can still be deleted', function () {
     makeOrphanFile('2026/07/empty.txt', '');