mirror of
https://github.com/projectsend/projectsend.git
synced 2026-10-06 13:21:56 +00:00
4b30849a88
The header checkbox on Import orphan files selected only the 25 rows on screen, so an install with thousands of stray files had to import them a page at a time. Once a whole page is ticked, the selection bar now offers "Select all N matching files", and "Import all" takes every orphan the search matches, on every page. The import runs in a queued job because it is too slow for a request. Each file is hashed in full and written in three commits, so 5,000 files of 4 MB take about four minutes, and PHP stops a request after 30 s of CPU, around file 1,100. ImportOrphanFilesJob works on the default queue in chunks of about 45 s: each chunk rescans, imports what is still orphaned and queues the next one. That keeps every job inside the worker's 60 s timeout and the queue's 90 s retry_after, so no extra worker is needed, and mail queued in the meantime goes out between chunks. If a run dies part way, the next one picks up what is left. Only one run can be active at a time. OrphanImportProgress keeps its state in the cache and starts a run under a lock. While a run is active, every other import is refused, the per-row button included, so no file is adopted twice. The page polls files/orphans/import-status every 3 s and shows the run as running, finished, failed with the reason, or stalled after 5 minutes without progress, which usually means no worker is listening. Bulk delete still works one page at a time. The adoption itself moved to OrphanFileImporter so the request and the job share it, and the rule for what can be imported now lives in OrphanFileScanner::importable().
132 lines
3.4 KiB
PHP
132 lines
3.4 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Modules\Files;
|
|
|
|
use Illuminate\Support\Facades\Cache;
|
|
|
|
/**
|
|
* The one background "import all" run on the orphans screen: how far it
|
|
* has got, and whether it is still going. Kept in the cache rather than a
|
|
* table: it is a progress readout for whoever is watching, not a record
|
|
* (every imported file is already in the activity log).
|
|
*
|
|
* There is only ever one run. Two at once would each scan the same disk
|
|
* and adopt the same paths twice, so a new one is refused while the
|
|
* current one is active.
|
|
*
|
|
* @phpstan-type Run array{status: string, total: int, imported: int, error: ?string, started_at: int, updated_at: int}
|
|
*/
|
|
class OrphanImportProgress
|
|
{
|
|
private const KEY = 'orphan-files:import-run';
|
|
|
|
/** A run kept for a day, so its outcome is still there after lunch. */
|
|
private const TTL_SECONDS = 86400;
|
|
|
|
/**
|
|
* Each chunk of the job runs for under a minute, and every imported
|
|
* file touches updated_at. A run silent for this long has no worker
|
|
* behind it (crashed, or no worker listening), and must not block
|
|
* the next one forever.
|
|
*/
|
|
private const STALL_SECONDS = 300;
|
|
|
|
/**
|
|
* Starts a run unless one is active, under a lock so two people
|
|
* clicking "Import all" at the same moment cannot both get one.
|
|
*/
|
|
public function tryStart(int $total): bool
|
|
{
|
|
return (bool) Cache::lock(self::KEY.':start', 10)->get(function () use ($total): bool {
|
|
if ($this->isActive()) {
|
|
return false;
|
|
}
|
|
|
|
$this->put([
|
|
'status' => 'running',
|
|
'total' => $total,
|
|
'imported' => 0,
|
|
'error' => null,
|
|
'started_at' => now()->getTimestamp(),
|
|
]);
|
|
|
|
return true;
|
|
});
|
|
}
|
|
|
|
public function advance(): void
|
|
{
|
|
$run = $this->raw();
|
|
|
|
if ($run !== null) {
|
|
$run['imported']++;
|
|
$this->put($run);
|
|
}
|
|
}
|
|
|
|
public function finish(): void
|
|
{
|
|
$this->end('finished');
|
|
}
|
|
|
|
public function fail(string $error): void
|
|
{
|
|
$this->end('failed', $error);
|
|
}
|
|
|
|
public function isActive(): bool
|
|
{
|
|
return ($this->current()['status'] ?? null) === 'running';
|
|
}
|
|
|
|
/**
|
|
* The run as the page shows it. A running run that has gone quiet is
|
|
* reported as stalled.
|
|
*
|
|
* @return array{status: string, total: int, imported: int, error: ?string, started_at: int}|null
|
|
*/
|
|
public function current(): ?array
|
|
{
|
|
$run = $this->raw();
|
|
|
|
if ($run === null) {
|
|
return null;
|
|
}
|
|
|
|
if ($run['status'] === 'running' && now()->getTimestamp() - $run['updated_at'] > self::STALL_SECONDS) {
|
|
$run['status'] = 'stalled';
|
|
}
|
|
|
|
unset($run['updated_at']);
|
|
|
|
return $run;
|
|
}
|
|
|
|
private function end(string $status, ?string $error = null): void
|
|
{
|
|
$run = $this->raw();
|
|
|
|
if ($run !== null) {
|
|
$this->put([...$run, 'status' => $status, 'error' => $error]);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @return Run|null
|
|
*/
|
|
private function raw(): ?array
|
|
{
|
|
return Cache::get(self::KEY);
|
|
}
|
|
|
|
/**
|
|
* @param array<string, mixed> $run
|
|
*/
|
|
private function put(array $run): void
|
|
{
|
|
Cache::put(self::KEY, [...$run, 'updated_at' => now()->getTimestamp()], self::TTL_SECONDS);
|
|
}
|
|
}
|