✨(backend) add file upload feature & tests

For the coming features we will need to store files on the meet side.
(for instance user backgrounds).

This commits adds a new Model to manage files, and the associated
serializers & viewsets. All are tested.

This work was heavily inspired by the work done by our friends at
https://github.com/suitenumerique/drive
It build on the same architecture design (upload directly to S3 but
download goes through our proxy), but model is much much simplier
(no folders, no file sharing, etc.).

src/backend/core/api/filters.py

@@ -0,0 +1,67 @@
+"""API filters for meet' core application."""
+
+from django.utils.translation import gettext_lazy as _
+
+import django_filters
+from django_filters import BooleanFilter
+
+from core import models
+
+
+class FileFilter(django_filters.FilterSet):
+    """
+    Custom filter for filtering files.
+    """
+
+    class Meta:
+        model = models.File
+        fields = ["type"]
+
+
+class ListFileFilter(FileFilter):
+    """Filter class dedicated to the file viewset list method."""
+
+    is_creator_me = django_filters.BooleanFilter(
+        method="filter_is_creator_me", label=_("Creator is me")
+    )
+
+    is_deleted = BooleanFilter(field_name="deleted_at", method="filter_is_deleted")
+
+    class Meta:
+        model = models.File
+        fields = ["is_creator_me", "type", "upload_state", "is_deleted"]
+
+    def filter_is_deleted(self, queryset, name, value):
+        """
+        Filter files based on whether they are deleted or not.
+
+        Example:
+            - /api/v1.0/files/?is_deleted=false
+                → Filters files that were not deleted
+        """
+        if value is None:
+            return queryset
+
+        lookup = "__".join([name, "isnull"])
+        return queryset.filter(**{lookup: not value})
+
+    # pylint: disable=unused-argument
+    def filter_is_creator_me(self, queryset, name, value):
+        """
+        Filter files based on the `creator` being the current user.
+
+        Example:
+            - /api/v1.0/files/?is_creator_me=true
+                → Filters files created by the logged-in user
+            - /api/v1.0/files/?is_creator_me=false
+                → Filters files created by other users
+        """
+        user = self.request.user
+
+        if not user.is_authenticated:
+            return queryset
+
+        if value:
+            return queryset.filter(creator=user)
+
+        return queryset.exclude(creator=user)

src/backend/core/api/permissions.py

@@ -1,5 +1,7 @@
 """Permission handlers for the Meet core app."""
 
+from django.http import Http404
+
 from rest_framework import permissions
 
 from ..models import RoleChoices
@@ -106,3 +108,23 @@ class HasLiveKitRoomAccess(permissions.BasePermission):
         if not request.auth or not hasattr(request.auth, "video"):
             return False
         return request.auth.video.room == str(obj.id)
+
+
+class FilePermission(IsAuthenticated):
+    """
+    Permissions applying to the file API endpoint.
+    Handling soft deletions specificities
+    """
+
+    def has_object_permission(self, request, view, obj):
+        """
+        Return a 404 on deleted files or if the user is not the owner
+        """
+
+        if obj.deleted_at is not None or obj.hard_deleted_at is not None:
+            raise Http404
+
+        if obj.creator != request.user:
+            raise Http404
+
+        return obj.get_abilities(request.user).get(view.action, False)

src/backend/core/api/serializers.py

@@ -1,10 +1,15 @@
 """Client serializers for the Meet core app."""
 
 # pylint: disable=abstract-method,no-name-in-module
+import logging
+from os.path import splitext
 from typing import Literal
+from urllib.parse import quote
 
 from django.conf import settings
 from django.core.exceptions import SuspiciousOperation
+
+# pylint: disable=abstract-method,no-name-in-module
 from django.utils.translation import gettext_lazy as _
 
 from django_pydantic_field.rest_framework import SchemaField
@@ -15,6 +20,8 @@ from timezone_field.rest_framework import TimeZoneSerializerField
 
 from core import models, utils
 
+logger = logging.getLogger(__name__)
+
 
 class UserSerializer(serializers.ModelSerializer):
     """Serialize users."""
@@ -27,6 +34,15 @@ class UserSerializer(serializers.ModelSerializer):
         read_only_fields = ["id", "email", "full_name", "short_name"]
 
 
+class UserLightSerializer(serializers.ModelSerializer):
+    """Serialize users with limited fields."""
+
+    class Meta:
+        model = models.User
+        fields = ["id", "full_name", "short_name"]
+        read_only_fields = ["id", "full_name", "short_name"]
+
+
 class ResourceAccessSerializerMixin:
     """
     A serializer mixin to share controlling that the logged-in user submitting a room access object
@@ -377,3 +393,143 @@ class UpdateParticipantSerializer(BaseParticipantsManagementSerializer):
             )
 
         return attrs
+
+
+class ListFileSerializer(serializers.ModelSerializer):
+    """Serialize File model for the API."""
+
+    url = serializers.SerializerMethodField(read_only=True)
+    creator = UserLightSerializer(read_only=True)
+    abilities = serializers.SerializerMethodField(read_only=True)
+
+    class Meta:
+        model = models.File
+        fields = [
+            "id",
+            "created_at",
+            "updated_at",
+            "title",
+            "type",
+            "creator",
+            "deleted_at",
+            "hard_deleted_at",
+            "filename",
+            "upload_state",
+            "mimetype",
+            "size",
+            "description",
+            "url",
+            "abilities",
+        ]
+        read_only_fields = [
+            "id",
+            "created_at",
+            "updated_at",
+            "creator",
+            "deleted_at",
+            "hard_deleted_at",
+            "filename",
+            "upload_state",
+            "mimetype",
+            "size",
+            "url",
+            "abilities",
+        ]
+
+    def get_url(self, obj):
+        """Return the URL of the file."""
+        if obj.is_pending_upload:
+            return None
+
+        return f"{settings.MEDIA_BASE_URL}{settings.MEDIA_URL}{quote(obj.file_key)}"
+
+    def get_abilities(self, file) -> dict:
+        """Return abilities of the logged-in user on the instance."""
+        request = self.context.get("request")
+        if not request:
+            return {}
+
+        return file.get_abilities(request.user)
+
+
+class FileSerializer(ListFileSerializer):
+    """Default serializer File model for the API."""
+
+    def create(self, validated_data):
+        raise NotImplementedError("Create method can not be used.")
+
+
+class CreateFileSerializer(ListFileSerializer):
+    """Serializer used to create a new file"""
+
+    title = serializers.CharField(max_length=255, required=False)
+    policy = serializers.SerializerMethodField()
+
+    class Meta:
+        model = models.File
+        fields = [*ListFileSerializer.Meta.fields, "policy"]
+        read_only_fields = [
+            *(
+                field
+                for field in ListFileSerializer.Meta.read_only_fields
+                if field != "filename"
+            ),
+            "policy",
+        ]
+
+    def get_fields(self):
+        """Force the id field to be writable."""
+        fields = super().get_fields()
+        fields["id"].read_only = False
+
+        return fields
+
+    def validate_id(self, value):
+        """Ensure the provided ID does not already exist when creating a new file."""
+        request = self.context.get("request")
+
+        # Only check this on POST (creation)
+        if request and models.File.objects.filter(id=value).exists():
+            raise serializers.ValidationError(
+                "A file with this ID already exists. You cannot override it.",
+                code="file_create_existing_id",
+            )
+
+        return value
+
+    def validate(self, attrs):
+        """Validate extension and fill title."""
+        # we run the default validation first to make sure the base data in attrs is ok
+        attrs = super().validate(attrs)
+
+        filename_root, ext = splitext(attrs["filename"])
+
+        if settings.FILE_UPLOAD_APPLY_RESTRICTIONS:
+            config_for_file_type = settings.FILE_UPLOAD_RESTRICTIONS[attrs["type"]]
+            if ext.lower() not in config_for_file_type["allowed_extensions"]:
+                logger.info(
+                    "create_item: file extension not allowed %s for filename %s",
+                    ext,
+                    attrs["filename"],
+                )
+                raise serializers.ValidationError(
+                    {"filename": _("This file extension is not allowed.")},
+                    code="item_create_file_extension_not_allowed",
+                )
+
+        # The title will be the filename if not provided
+        if not attrs.get("title", None):
+            attrs["title"] = filename_root
+
+        return attrs
+
+    def get_policy(self, file):
+        """Return the policy to use if the item is a file."""
+
+        if file.upload_state == models.FileUploadStateChoices.READY:
+            return None
+
+        return utils.generate_upload_policy(file)
+
+    def update(self, instance, validated_data):
+        raise NotImplementedError("Update method can not be used.")

src/backend/core/api/viewsets.py

@@ -1,16 +1,26 @@
 """API endpoints"""
+# pylint: disable=too-many-lines
 
+import re
 import uuid
 from logging import getLogger
-from urllib.parse import urlparse
+from urllib.parse import unquote, urlparse
 
 from django.conf import settings
+from django.core.files.storage import default_storage
 from django.db.models import Q
 from django.http import Http404
 from django.shortcuts import get_object_or_404
 from django.utils.text import slugify
 
-from rest_framework import decorators, mixins, pagination, viewsets
+from django_filters import rest_framework as django_filters
+from rest_framework import (
+    decorators,
+    filters,
+    mixins,
+    pagination,
+    viewsets,
+)
 from rest_framework import (
     exceptions as drf_exceptions,
 )
@@ -22,6 +32,7 @@ from rest_framework import (
 )
 
 from core import enums, models, utils
+from core.api.filters import ListFileFilter
 from core.recording.enums import FileExtension
 from core.recording.event.authentication import StorageEventAuthentication
 from core.recording.event.exceptions import (
@@ -56,6 +67,7 @@ from core.services.participants_management import (
 )
 from core.services.room_creation import RoomCreation
 from core.services.subtitle import SubtitleException, SubtitleService
+from core.tasks.file import process_file_deletion
 
 from ..authentication.livekit import LiveKitTokenAuthentication
 from . import permissions, serializers, throttling
@@ -66,6 +78,17 @@ from .feature_flag import FeatureFlag
 logger = getLogger(__name__)
 
 
+FILE_FOLDER = settings.FILE_UPLOAD_PATH
+UUID_REGEX = (
+    r"[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"
+)
+FILE_EXT_REGEX = r"[\d\w]+"
+MEDIA_STORAGE_URL_PATTERN = re.compile(
+    f"{settings.MEDIA_URL:s}"
+    rf"(?P<key>{FILE_FOLDER:s}/(?P<pk>{UUID_REGEX:s})/\.{FILE_EXT_REGEX:s})$"
+)
+
+
 class NestedGenericViewSet(viewsets.GenericViewSet):
     """
     A generic Viewset aims to be used in a nested route context.
@@ -77,20 +100,20 @@ class NestedGenericViewSet(viewsets.GenericViewSet):
     lookup_fields: list[str] = ["pk"]
     lookup_url_kwargs: list[str] = []
 
-    def __getattribute__(self, item):
+    def __getattribute__(self, file):
         """
         This method is overridden to allow to get the last lookup field or lookup url kwarg
         when accessing the `lookup_field` or `lookup_url_kwarg` attribute. This is useful
         to keep compatibility with all methods used by the parent class `GenericViewSet`.
         """
-        if item in ["lookup_field", "lookup_url_kwarg"]:
-            return getattr(self, item + "s", [None])[-1]
+        if file in ["lookup_field", "lookup_url_kwarg"]:
+            return getattr(self, file + "s", [None])[-1]
 
-        return super().__getattribute__(item)
+        return super().__getattribute__(file)
 
     def get_queryset(self):
         """
-        Get the list of items for this view.
+        Get the list of files for this view.
 
         `lookup_fields` attribute is enumerated here to perform the nested lookup.
         """
@@ -793,7 +816,7 @@ class RecordingViewSet(
         # Extract the original URL from the request header
         original_url = request.META.get("HTTP_X_ORIGINAL_URL")
         if not original_url:
-            logger.debug("Missing HTTP_X_ORIGINAL_URL header in subrequest")
+            logger.warning("Missing HTTP_X_ORIGINAL_URL header in subrequest")
             raise drf_exceptions.PermissionDenied()
 
         logger.debug("Original url: '%s'", original_url)
@@ -810,7 +833,7 @@ class RecordingViewSet(
         try:
             return match.groupdict()
         except (ValueError, AttributeError) as exc:
-            logger.debug("Failed to extract parameters from subrequest URL: %s", exc)
+            logger.warning("Failed to extract parameters from subrequest URL: %s", exc)
             raise drf_exceptions.PermissionDenied() from exc
 
     @decorators.action(detail=False, methods=["get"], url_path="media-auth")
@@ -834,7 +857,7 @@ class RecordingViewSet(
         recording_id = url_params["recording_id"]
 
         extension = url_params["extension"]
-        if extension not in [item.value for item in FileExtension]:
+        if extension not in [file.value for file in FileExtension]:
             raise drf_exceptions.ValidationError({"detail": "Unsupported extension."})
 
         try:
@@ -858,3 +881,302 @@ class RecordingViewSet(
         request = utils.generate_s3_authorization_headers(recording.key)
 
         return drf_response.Response("authorized", headers=request.headers, status=200)
+
+
+# pylint: disable=too-many-public-methods
+class FileViewSet(
+    SerializerPerActionMixin,
+    mixins.CreateModelMixin,
+    mixins.DestroyModelMixin,
+    mixins.UpdateModelMixin,
+    mixins.ListModelMixin,
+    viewsets.GenericViewSet,
+):
+    """
+    FileViewSet API.
+
+    This viewset provides CRUD operations and additional actions for managing files.
+
+    ### API Endpoints:
+    1. **List**: Retrieve a paginated list of files.
+       Example: GET /files/?page=2
+    2. **Retrieve**: Get a specific file by its ID.
+       Example: GET /files/{id}/
+    3. **Create**: Create a new file.
+       Example: POST /files/
+    4. **Update**: Update a file by its ID.
+       Example: PUT /files/{id}/
+    5. **Delete**: Soft delete a file by its ID.
+       Example: DELETE /files/{id}/
+
+
+    ### Ordering: created_at, updated_at, title
+
+        Example:
+        - Ascending: GET /api/v1.0/files/?ordering=created_at
+
+    ### Filtering:
+        - `is_creator_me=true`: Returns files created by the current user.
+        - `is_creator_me=false`: Returns files created by other users.
+        - `is_deleted=false`: Returns files that are not (soft) deleted
+
+        Example:
+        - GET /api/v1.0/files/?is_creator_me=true
+        - GET /api/v1.0/files/?is_creator_me=false&is_deleted=false
+
+    ### Notes:
+    - Implements soft delete logic to retain file
+    """
+
+    ordering = ["-updated_at"]
+    ordering_fields = ["created_at", "updated_at", "title"]
+    pagination_class = Pagination
+    permission_classes = [
+        permissions.FilePermission,
+    ]
+    queryset = models.File.objects.filter(hard_deleted_at__isnull=True)
+    default_serializer_class = serializers.FileSerializer
+    serializer_classes = {
+        "list": serializers.ListFileSerializer,
+        "create": serializers.CreateFileSerializer,
+    }
+    filter_backends = (django_filters.DjangoFilterBackend, filters.OrderingFilter)
+    filterset_class = ListFileFilter
+
+    def get_queryset(self):
+        """Get queryset that defaults to the the current request user."""
+        user = self.request.user
+        queryset = super().get_queryset().select_related("creator")
+
+        if not user.is_authenticated:
+            return queryset.none()
+
+        # For now, we force the filtering on the current user in all cases, might evolve later
+        queryset = queryset.filter(creator=user)
+        return queryset
+
+    def get_response_for_queryset(self, queryset, context=None):
+        """Return paginated response for the queryset if requested."""
+        context = context or self.get_serializer_context()
+        page = self.paginate_queryset(queryset)
+        if page is not None:
+            serializer = self.get_serializer(page, many=True, context=context)
+            result = self.get_paginated_response(serializer.data)
+            return result
+
+        serializer = self.get_serializer(queryset, many=True, context=context)
+        return drf_response.Response(serializer.data)
+
+    def perform_create(self, serializer):
+        """Set the current user as creator of the newly created file."""
+        serializer.save(creator=self.request.user)
+
+    def perform_destroy(self, instance):
+        """Override to implement a soft delete instead of dumping the record in database."""
+        instance.soft_delete()
+
+    @decorators.action(detail=True, methods=["post"], url_path="upload-ended")
+    def upload_ended(self, request, *args, **kwargs):
+        """
+        Check the actual uploaded file and mark it as ready.
+        """
+
+        file = self.get_object()
+
+        if not file.is_pending_upload:
+            raise drf_exceptions.ValidationError(
+                {"file": "This action is only available for files in PENDING state."},
+                code="file_upload_state_not_pending",
+            )
+
+        s3_client = default_storage.connection.meta.client
+
+        head_response = s3_client.head_object(
+            Bucket=default_storage.bucket_name, Key=file.file_key
+        )
+        file_size = head_response["ContentLength"]
+
+        if settings.FILE_UPLOAD_APPLY_RESTRICTIONS:
+            config_for_file_type = settings.FILE_UPLOAD_RESTRICTIONS[file.type]
+            if file_size > config_for_file_type["max_size"]:
+                self._complete_file_deletion(file)
+                logger.info(
+                    "upload_ended: file size (%s) for file %s higher than the allowed max size",
+                    file_size,
+                    file.file_key,
+                )
+                raise drf_exceptions.ValidationError(
+                    detail="The file size is higher than the allowed max size.",
+                    code="file_size_exceeded",
+                )
+
+        # python-magic recommends using at least the first 2048 bytes
+        # to reduce incorrect identification.
+        # This is a tradeoff between pulling in the whole file and the most likely relevant bytes
+        # of the file for mime type identification.
+        if file_size > 2048:
+            range_response = s3_client.get_object(
+                Bucket=default_storage.bucket_name,
+                Key=file.file_key,
+                Range="bytes=0-2047",
+            )
+            file_head = range_response["Body"].read()
+        else:
+            file_head = s3_client.get_object(
+                Bucket=default_storage.bucket_name, Key=file.file_key
+            )["Body"].read()
+
+        # Use improved MIME type detection combining magic bytes and file extension
+        logger.info("upload_ended: detecting mimetype for file: %s", file.file_key)
+        mimetype = utils.detect_mimetype(file_head, filename=file.filename)
+
+        if settings.FILE_UPLOAD_APPLY_RESTRICTIONS:
+            config_for_file_type = settings.FILE_UPLOAD_RESTRICTIONS[file.type]
+            allowed_file_mimetypes = config_for_file_type["allowed_mimetypes"]
+            if mimetype not in allowed_file_mimetypes:
+                self._complete_file_deletion(file)
+                logger.warning(
+                    "upload_ended: mimetype not allowed %s for file %s",
+                    mimetype,
+                    file.file_key,
+                )
+                raise drf_exceptions.ValidationError(
+                    detail="The file type is not allowed.",
+                    code="file_type_not_allowed",
+                )
+
+        file.upload_state = models.FileUploadStateChoices.READY
+        file.mimetype = mimetype
+        file.size = file_size
+
+        file.save(update_fields=["upload_state", "mimetype", "size"])
+
+        if head_response["ContentType"] != mimetype:
+            logger.info(
+                "upload_ended: content type mismatch between object storage and file,"
+                " updating from %s to %s",
+                head_response["ContentType"],
+                mimetype,
+            )
+            s3_client.copy_object(
+                Bucket=default_storage.bucket_name,
+                Key=file.file_key,
+                CopySource={
+                    "Bucket": default_storage.bucket_name,
+                    "Key": file.file_key,
+                },
+                ContentType=mimetype,
+                Metadata=head_response["Metadata"],
+                MetadataDirective="REPLACE",
+            )
+
+        # Not yet implemented
+        # Change the file.upload_state when this will be done
+        # malware_detection.analyse_file(file.file_key, file_id=file.id)
+
+        serializer = self.get_serializer(file)
+
+        return drf_response.Response(serializer.data, status=drf_status.HTTP_200_OK)
+
+    def _complete_file_deletion(self, file):
+        """Delete a file completely."""
+        file.soft_delete()
+        file.hard_delete()
+        process_file_deletion.delay(file.id)
+
+    def _authorize_subrequest(self, request, pattern):
+        """
+        Authorize access based on the original URL of an Nginx subrequest
+        and user permissions. Returns a dictionary of URL parameters if authorized.
+
+        The original url is passed by nginx in the "HTTP_X_ORIGINAL_URL" header.
+        See corresponding ingress configuration in Helm chart and read about the
+        nginx.ingress.kubernetes.io/auth-url annotation to understand how the Nginx ingress
+        is configured to do this.
+
+        Based on the original url and the logged in user, we must decide if we authorize Nginx
+        to let this request go through (by returning a 200 code) or if we block it (by returning
+        a 403 error). Note that we return 403 errors without any further details for security
+        reasons.
+
+        Parameters:
+        - pattern: The regex pattern to extract identifiers from the URL.
+
+        Returns:
+        - A dictionary of URL parameters if the request is authorized.
+        Raises:
+        - PermissionDenied if authorization fails.
+        """
+        # Extract the original URL from the request header
+        original_url = request.META.get("HTTP_X_ORIGINAL_URL")
+        if not original_url:
+            logger.warning("Missing HTTP_X_ORIGINAL_URL header in subrequest")
+            raise drf_exceptions.PermissionDenied()
+
+        parsed_url = urlparse(original_url)
+        match = pattern.search(unquote(parsed_url.path))
+
+        if not match:
+            logger.warning(
+                "Subrequest URL '%s' did not match pattern '%s'",
+                parsed_url.path,
+                pattern,
+            )
+            raise drf_exceptions.PermissionDenied()
+
+        try:
+            url_params = match.groupdict()
+        except (ValueError, AttributeError) as exc:
+            logger.warning("Failed to extract parameters from subrequest URL: %s", exc)
+            raise drf_exceptions.PermissionDenied() from exc
+
+        pk = url_params.get("pk")
+        if not pk:
+            logger.warning("File ID (pk) not found in URL parameters: %s", url_params)
+            raise drf_exceptions.PermissionDenied()
+
+        # Fetch the file and check if the user has access
+        queryset = models.File.objects.all()
+        # No suspicious analysis implemented yet
+        # queryset = self._filter_suspicious_files(queryset, request.user)
+        try:
+            file = queryset.get(pk=pk)
+        except models.File.DoesNotExist as exc:
+            logger.warning("File with ID '%s' does not exist", pk)
+            raise drf_exceptions.PermissionDenied() from exc
+
+        user_abilities = file.get_abilities(request.user)
+        if not user_abilities.get(self.action, False):
+            logger.warning(
+                "User '%s' lacks permission for file '%s'", request.user.id, pk
+            )
+            raise drf_exceptions.PermissionDenied()
+
+        logger.debug(
+            "Subrequest authorization successful. Extracted parameters: %s", url_params
+        )
+        return url_params, request.user.id, file
+
+    @decorators.action(detail=False, methods=["get"], url_path="media-auth")
+    def media_auth(self, request, *args, **kwargs):
+        """
+        This view is used by an Nginx subrequest to control access to an file's
+        attachment file.
+
+        When we let the request go through, we compute authorization headers that will be added to
+        the request going through thanks to the nginx.ingress.kubernetes.io/auth-response-headers
+        annotation. The request will then be proxied to the object storage backend who will
+        respond with the file after checking the signature included in headers.
+        """
+        url_params, _, file = self._authorize_subrequest(
+            request, MEDIA_STORAGE_URL_PATTERN
+        )
+
+        if file.is_pending_upload:
+            logger.warning("File '%s' is not ready", file.id)
+            raise drf_exceptions.PermissionDenied()
+
+        # Generate S3 authorization headers using the extracted URL parameters
+        request = utils.generate_s3_authorization_headers(f"{url_params.get('key'):s}")
+
+        return drf_response.Response("authorized", headers=request.headers, status=200)