From 5c151407831562f8a58abb0ae4d2ceae96d06b3a Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Wed, 3 Jun 2026 17:38:09 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(backend)=20add=20command=20to=20clean?= =?UTF-8?q?=20pending=20and=20deleted=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since we have added custom backgrounds, they could be soft deleted but were never really deleted. This commits introduces a command to actually delete this files. In introduces also a command to purge pending upload . --- CHANGELOG.md | 5 ++ src/backend/core/api/viewsets.py | 5 +- .../commands/clean_pending_files.py | 47 ++++++++++ .../commands/purge_deleted_files.py | 40 +++++++++ src/backend/core/tests/commands/__init__.py | 0 .../commands/test_clean_pending_files.py | 90 +++++++++++++++++++ .../commands/test_purge_deleted_files.py | 85 ++++++++++++++++++ src/backend/meet/settings.py | 4 +- 8 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 src/backend/core/management/commands/clean_pending_files.py create mode 100644 src/backend/core/management/commands/purge_deleted_files.py create mode 100644 src/backend/core/tests/commands/__init__.py create mode 100644 src/backend/core/tests/commands/test_clean_pending_files.py create mode 100644 src/backend/core/tests/commands/test_purge_deleted_files.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 32771f32..787a817d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to ## [Unreleased] +### Added + +- ✨(backend) add command to clean pending and deleted files +- 🧱(helm) run clean files command as cronjob + ## [1.21.0] - 2026-06-15 ### Added diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 242d8255..e731cf42 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -1187,7 +1187,10 @@ class FileViewSet( serializer.save(creator=self.request.user) def perform_destroy(self, instance): - """Override to implement a soft delete instead of dumping the record in database.""" + """Override to implement a soft delete instead of dumping the record in database. + + Files are actually purged by commands that should run periodically. + """ instance.soft_delete() @decorators.action(detail=True, methods=["post"], url_path="upload-ended") diff --git a/src/backend/core/management/commands/clean_pending_files.py b/src/backend/core/management/commands/clean_pending_files.py new file mode 100644 index 00000000..9a8b8041 --- /dev/null +++ b/src/backend/core/management/commands/clean_pending_files.py @@ -0,0 +1,47 @@ +"""Clean stale pending files that were never fully uploaded.""" + +from datetime import timedelta + +from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone + +from core.models import File, FileUploadStateChoices +from core.tasks.file import process_file_deletion + + +class Command(BaseCommand): + """Remove pending files older than a given threshold.""" + + help = "Delete pending files that have been stuck for too long" + + def add_arguments(self, parser): + parser.add_argument( + "--hours", + type=int, + default=24, + help="Age threshold in hours (default: 24)", + ) + + def handle(self, *args, **options): + hours = options["hours"] + if hours < 0: + raise CommandError("Hours must be greater than 0") + + threshold = timezone.now() - timedelta(hours=hours) + + files = File.objects.filter( + upload_state=FileUploadStateChoices.PENDING, + created_at__lt=threshold, + hard_deleted_at__isnull=True, + ) + + count = 0 + for file in files.iterator(): + # This check shouldn't happen, but just in case we do it to avoid an error + if not file.deleted_at: + file.soft_delete() + file.hard_delete() + process_file_deletion(file.id) + count += 1 + + self.stdout.write(f"Cleaned {count} stale pending file(s).") diff --git a/src/backend/core/management/commands/purge_deleted_files.py b/src/backend/core/management/commands/purge_deleted_files.py new file mode 100644 index 00000000..10d71516 --- /dev/null +++ b/src/backend/core/management/commands/purge_deleted_files.py @@ -0,0 +1,40 @@ +"""Purge deleted files.""" + +from datetime import timedelta + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.db.models import Q +from django.utils import timezone + +from core.models import File +from core.tasks.file import process_file_deletion + + +class Command(BaseCommand): + """ + Purge deleted files (object storage and database object): + - files marked as hard deleted in database + - files marked as soft deleted and for which the trashbin retention period has expired + """ + + help = "Purge deleted files" + + def handle(self, *args, **options): + """Browse purgeable files and queue them through the file deletion task.""" + + is_hard_deleted = Q(hard_deleted_at__isnull=False) + is_purgeable = Q( + deleted_at__lte=timezone.now() + - timedelta(days=settings.FILE_PURGE_GRACE_DAYS) + ) + + count = 0 + for file in File.objects.filter(is_hard_deleted | is_purgeable).iterator(): + if file.hard_deleted_at is None: + file.hard_delete() + + process_file_deletion.delay(file.id) + count += 1 + + self.stdout.write(f"Purged {count} deleted file(s).") diff --git a/src/backend/core/tests/commands/__init__.py b/src/backend/core/tests/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/core/tests/commands/test_clean_pending_files.py b/src/backend/core/tests/commands/test_clean_pending_files.py new file mode 100644 index 00000000..264f4dbc --- /dev/null +++ b/src/backend/core/tests/commands/test_clean_pending_files.py @@ -0,0 +1,90 @@ +"""Tests for the clean_pending_files management command.""" + +from datetime import timedelta + +from django.core.files.storage import default_storage +from django.core.management import call_command +from django.utils import timezone + +import pytest + +from core import factories, models + +pytestmark = pytest.mark.django_db + + +def test_clean_pending_files_no_stale_files(): + """Nothing happens when there are no stale pending files.""" + call_command("clean_pending_files") + + +def test_clean_pending_files_recent_pending_not_deleted(): + """Recent pending files (within threshold) should not be deleted.""" + file = factories.FileFactory( + type=models.FileTypeChoices.BACKGROUND_IMAGE, + update_upload_state=models.FileUploadStateChoices.PENDING, + upload_bytes=b"hello", + ) + + call_command("clean_pending_files") + + file.refresh_from_db() + assert file.deleted_at is None + assert default_storage.exists(file.file_key) + + +def test_clean_pending_files_old_pending_deleted(): + """Pending files older than the threshold should be deleted.""" + old_date = timezone.now() - timedelta(hours=49) + file = factories.FileFactory( + type=models.FileTypeChoices.BACKGROUND_IMAGE, + update_upload_state=models.FileUploadStateChoices.PENDING, + upload_bytes=b"hello", + ) + assert default_storage.exists(file.file_key) + models.File.objects.filter(pk=file.pk).update(created_at=old_date) + + call_command("clean_pending_files") + + assert not models.File.objects.filter(pk=file.pk).exists() + assert not default_storage.exists(file.file_key) + + +def test_clean_pending_files_old_non_pending_not_deleted(): + """Old files that are not pending should not be deleted.""" + old_date = timezone.now() - timedelta(hours=49) + file = factories.FileFactory( + type=models.FileTypeChoices.BACKGROUND_IMAGE, + update_upload_state=models.FileUploadStateChoices.READY, + ) + models.File.objects.filter(pk=file.pk).update(created_at=old_date) + + call_command("clean_pending_files") + + file.refresh_from_db() + assert file.deleted_at is None + assert file.hard_deleted_at is None + + +def test_clean_pending_files_custom_hours(): + """The --hours argument controls the age threshold.""" + old_date = timezone.now() - timedelta(hours=10) + file = factories.FileFactory( + type=models.FileTypeChoices.BACKGROUND_IMAGE, + update_upload_state=models.FileUploadStateChoices.PENDING, + upload_bytes=b"hello", + ) + models.File.objects.filter(pk=file.pk).update(created_at=old_date) + + # Default 24h threshold -> file not deleted + call_command("clean_pending_files") + + file.refresh_from_db() + assert file.deleted_at is None + assert default_storage.exists(file.file_key) + + # 8h threshold -> file deleted + call_command("clean_pending_files", "--hours=8") + + assert not models.File.objects.filter(pk=file.pk).exists() + assert not default_storage.exists(file.file_key) diff --git a/src/backend/core/tests/commands/test_purge_deleted_files.py b/src/backend/core/tests/commands/test_purge_deleted_files.py new file mode 100644 index 00000000..28e51605 --- /dev/null +++ b/src/backend/core/tests/commands/test_purge_deleted_files.py @@ -0,0 +1,85 @@ +"""Tests for the purge_deleted_files management command.""" + +from datetime import timedelta +from io import StringIO +from random import randint +from unittest.mock import patch + +from django.core.files.storage import default_storage +from django.core.management import call_command +from django.utils import timezone + +import pytest + +from core import factories, models +from core.tasks.file import process_file_deletion + +pytestmark = pytest.mark.django_db + + +def test_purge_deleted_files_no_deleted_files(django_assert_num_queries): + """Nothing happens when there are no purgeable files.""" + with django_assert_num_queries(1): + call_command("purge_deleted_files") + + +@pytest.mark.django_db(transaction=True) +def test_purge_deleted_files_success(settings): + """ + Queue deletion for: + - hard-deleted files + - soft-deleted files past retention period + grace period. + """ + out = StringIO() + + settings.FILE_PURGE_GRACE_DAYS = grace = randint(1, 20) + + now = timezone.now() + purge_now = now - timedelta(days=grace) + + not_deleted_file = factories.FileFactory( + type=models.FileTypeChoices.BACKGROUND_IMAGE, + upload_bytes=b"hello", + ) + + with patch("django.utils.timezone.now", return_value=now): + not_purgeable_file = factories.FileFactory( + type=models.FileTypeChoices.BACKGROUND_IMAGE, + upload_bytes=b"hello", + ) + not_purgeable_file.soft_delete() + + with patch("django.utils.timezone.now", return_value=purge_now): + purgeable_file = factories.FileFactory( + type=models.FileTypeChoices.BACKGROUND_IMAGE, + upload_bytes=b"hello", + ) + purgeable_file.soft_delete() + + hard_deleted_file = factories.FileFactory( + type=models.FileTypeChoices.BACKGROUND_IMAGE, + upload_bytes=b"hello", + ) + hard_deleted_file.soft_delete() + hard_deleted_file.hard_delete() + + with patch( + "core.management.commands.purge_deleted_files.process_file_deletion.delay", + side_effect=process_file_deletion, + ) as mock_delay: + call_command("purge_deleted_files", stdout=out) + + assert "Purged 2 deleted file(s)." in out.getvalue() + assert mock_delay.call_count == 2 + called_ids = {call.args[0] for call in mock_delay.call_args_list} + assert called_ids == {purgeable_file.id, hard_deleted_file.id} + + assert models.File.objects.filter(id=not_deleted_file.id).exists() + assert models.File.objects.filter(id=not_purgeable_file.id).exists() + assert not models.File.objects.filter(id=purgeable_file.id).exists() + assert not models.File.objects.filter(id=hard_deleted_file.id).exists() + + assert default_storage.exists(not_deleted_file.file_key) + assert default_storage.exists(not_purgeable_file.file_key) + assert not default_storage.exists(purgeable_file.file_key) + assert not default_storage.exists(hard_deleted_file.file_key) diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.py index 05ce3edb..72d6fbd5 100755 --- a/src/backend/meet/settings.py +++ b/src/backend/meet/settings.py @@ -182,7 +182,9 @@ class Base(Configuration): environ_name="FILE_UPLOAD_ENABLED", environ_prefix=None, ) - + FILE_PURGE_GRACE_DAYS = values.PositiveIntegerValue( + default=7, environ_name="FILE_PURGE_GRACE_DAYS", environ_prefix=None + ) FILE_UPLOAD_PATH = values.Value( "files", environ_name="FILE_UPLOAD_PATH", environ_prefix=None )