diff --git a/Dockerfile b/Dockerfile index 97db272c..61b2b4c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,7 @@ ARG MEET_STATIC_ROOT=/data/static RUN apk add \ pango \ + libmagic \ rdfind # Copy installed python dependencies @@ -68,6 +69,7 @@ RUN apk --no-cache add \ gettext \ libffi-dev \ pango \ + libmagic \ shared-mime-info diff --git a/src/backend/core/tests/test_utils_detect_mimetype.py b/src/backend/core/tests/test_utils_detect_mimetype.py new file mode 100644 index 00000000..13abf821 --- /dev/null +++ b/src/backend/core/tests/test_utils_detect_mimetype.py @@ -0,0 +1,166 @@ +""" +Test utils.detect_mimetype +Originally taken from https://github.com/suitenumerique/drive/blob/564822d31f071c6dfacd112ef4b7146c73077cd9/src/backend/core/api/utils.py#L166 # pylint:disable=line-too-long +""" + +import pytest + +from core import utils + + +def test_detect_mimetype_from_content_pdf(): + """Test detect_mimetype detects PDF from content (magic bytes).""" + # PDF magic bytes: %PDF + pdf_content = b"%PDF-1.4\n" + mimetype = utils.detect_mimetype(pdf_content, filename="document.pdf") + assert mimetype == "application/pdf" + + +def test_detect_mimetype_from_content_png(): + """Test detect_mimetype detects PNG from content (magic bytes).""" + # PNG magic bytes: \x89PNG\r\n\x1a\n + png_content = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR" + mimetype = utils.detect_mimetype(png_content, filename="image.png") + assert mimetype == "image/png" + + +def test_detect_mimetype_from_content_jpeg(): + """Test detect_mimetype detects JPEG from content (magic bytes).""" + # JPEG magic bytes: \xff\xd8\xff + jpeg_content = b"\xff\xd8\xff\xe0\x00\x10JFIF" + mimetype = utils.detect_mimetype(jpeg_content, filename="photo.jpg") + assert mimetype == "image/jpeg" + + +def test_detect_mimetype_from_content_text_plain(): + """Test detect_mimetype detects plain text from content.""" + text_content = b"This is plain text content" + mimetype = utils.detect_mimetype(text_content, filename="file.txt") + assert mimetype == "text/plain" + + +def test_detect_mimetype_empty_file(): + """Test detect_mimetype handles empty files.""" + empty_content = b"" + mimetype = utils.detect_mimetype(empty_content, filename="empty.txt") + # Empty files should be detected as application/x-empty by magic bytes + assert mimetype == "application/x-empty" + + +def test_detect_mimetype_uses_extension_when_content_generic(): + """Test detect_mimetype uses extension when content detection returns generic type.""" + # Generic binary content that might be detected as application/octet-stream + generic_content = b"\x00\x01\x02\x03\x04\x05" + # But if we have a specific extension, we should use it + mimetype = utils.detect_mimetype(generic_content, filename="document.pdf") + # Should prefer extension-based detection for specific types + assert mimetype == "application/pdf" + + +def test_detect_mimetype_uses_extension_for_json(): + """Test detect_mimetype uses extension for JSON files.""" + # JSON content might be detected as text/plain + json_content = b'{"key": "value"}' + mimetype = utils.detect_mimetype(json_content, filename="data.json") + # Should use extension to get application/json + assert mimetype == "application/json" + + +def test_detect_mimetype_without_filename(): + """Test detect_mimetype works without filename (content-only detection).""" + pdf_content = b"%PDF-1.4\n" + mimetype = utils.detect_mimetype(pdf_content, filename=None) + assert mimetype == "application/pdf" + + +def test_detect_mimetype_fallback_to_extension(): + """Test detect_mimetype falls back to extension when content detection is generic.""" + # Content that might be detected as text/plain + content = b"some content" + mimetype = utils.detect_mimetype(content, filename="script.js") + # Should use extension to get JavaScript MIME type + # (can be text/javascript or application/javascript) + assert mimetype in ["text/javascript", "application/javascript"] + + +def test_detect_mimetype_generic_content_no_extension(): + """Test detect_mimetype with generic content and no extension.""" + generic_content = b"\x00\x01\x02\x03" + mimetype = utils.detect_mimetype(generic_content, filename="file") + # Should return content-based detection (likely application/octet-stream) + assert mimetype in ["application/octet-stream", "application/x-empty"] + + +def test_detect_mimetype_xml_file(): + """Test detect_mimetype detects XML files.""" + xml_content = b'' + mimetype = utils.detect_mimetype(xml_content, filename="data.xml") + # Should detect as XML (either from content or extension) + assert mimetype in ["application/xml", "text/xml"] + + +def test_detect_mimetype_csv_file(): + """Test detect_mimetype detects CSV files.""" + csv_content = b"name,age\nJohn,30\nJane,25" + mimetype = utils.detect_mimetype(csv_content, filename="data.csv") + # CSV might be detected as text/plain, but extension should help + assert mimetype in ["text/csv", "text/plain"] + + +def test_detect_mimetype_zip_file(): + """Test detect_mimetype detects ZIP files from magic bytes.""" + # ZIP magic bytes: PK\x03\x04 + zip_content = b"PK\x03\x04\x14\x00\x00\x00" + mimetype = utils.detect_mimetype(zip_content, filename="archive.zip") + assert mimetype == "application/zip" + + +def test_detect_mimetype_prefers_content_over_extension(): + """Test detect_mimetype prefers content detection when both are available and specific.""" + # PDF content but wrong extension + pdf_content = b"%PDF-1.4\n" + mimetype = utils.detect_mimetype(pdf_content, filename="document.txt") + # Should prefer content detection (PDF) over extension (txt) + assert mimetype == "application/pdf" + + +@pytest.mark.xfail( + reason="Fails in our repo, but passes in the original repo, leaving it there" +) +def test_detect_mimetype_powerpoint_pptx(): + """Test detect_mimetype correctly detects PowerPoint .pptx files.""" + # .pptx files are ZIP archives, so content might be detected as application/zip or octet-stream + # But with the extension, it should be detected as PowerPoint MIME type + # Using minimal ZIP-like content that might be detected as generic + pptx_content = b"PK\x03\x04" # ZIP magic bytes (PPTX is a ZIP archive) + mimetype = utils.detect_mimetype(pptx_content, filename="presentation.pptx") + # Should use extension to get PowerPoint MIME type + assert mimetype in [ + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + ] + + +def test_detect_mimetype_powerpoint_ppt(): + """Test detect_mimetype correctly detects PowerPoint .ppt files (older format).""" + # .ppt files might be detected as application/octet-stream by magic bytes + # But with the extension, it should be detected as PowerPoint MIME type + ppt_content = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" # OLE2 compound document header + mimetype = utils.detect_mimetype(ppt_content, filename="presentation.ppt") + # Should use extension to get PowerPoint MIME type + assert mimetype in [ + "application/vnd.ms-powerpoint", + "application/mspowerpoint", + ] + + +def test_detect_mimetype_ole_storage(): + """ + Microsoft files .xls .doc .ppt can use the mimetype application/x-ole-storage + for Microsoft OLE2/Compound File Binary Format. In that case we want to rely on the extension + """ + + xls_ole_content = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + mimetype = utils.detect_mimetype(xls_ole_content, filename="document.xls") + + assert mimetype == "application/vnd.ms-excel" diff --git a/src/backend/core/utils.py b/src/backend/core/utils.py index 5079e2c1..f4403660 100644 --- a/src/backend/core/utils.py +++ b/src/backend/core/utils.py @@ -7,6 +7,8 @@ Utils functions used in the core app import hashlib import json +import logging +import mimetypes import random import secrets import string @@ -18,6 +20,7 @@ from django.core.files.storage import default_storage import aiohttp import botocore +import magic from asgiref.sync import async_to_sync from livekit.api import ( # pylint: disable=E0611 AccessToken, @@ -29,6 +32,8 @@ from livekit.api import ( # pylint: disable=E0611 VideoGrants, ) +logger = logging.getLogger(__name__) + def generate_color(identity: str) -> str: """Generates a consistent HSL color based on a given identity string. @@ -344,3 +349,65 @@ def generate_room_slug(): for size in sizes ] return "-".join(parts) + + +def detect_mimetype(file_buffer: bytes, filename: str | None = None) -> str: + """ + Detect MIME type using multiple methods for better accuracy. + + This function combines: + 1. Magic bytes detection (python-magic) - most reliable for actual file content + 2. File extension detection (mimetypes) - useful as fallback or for validation + + Args: + file_buffer: The file content buffer (first bytes of the file) + filename: Optional filename to extract extension from + + Returns: + str: The detected MIME type + + Notes: + Originally from https://github.com/suitenumerique/drive/blob/564822d31f071c6dfacd112ef4b7146c73077cd9/src/backend/core/api/utils.py#L166 # pylint:disable=line-too-long + """ + # Initialize magic detector + mime_detector = magic.Magic(mime=True) + + # Method 1: Detect from file content (magic bytes) - most reliable + mimetype_from_content = mime_detector.from_buffer(file_buffer) + + # If we have a filename, try extension-based detection as well + mimetype_from_extension = None + if filename: + # Use mimetypes module to guess from extension + # Use guess_file_type (Python 3.13+) instead of deprecated guess_type + mimetype_from_extension, _ = mimetypes.guess_file_type(filename, strict=False) + + logger.info("detect_mimetype: mimetype_from_content: %s", mimetype_from_content) + logger.info("detect_mimetype: mimetype_from_extension: %s", mimetype_from_extension) + + # Strategy: Prefer content-based detection, but use extension if: + # 1. Content detection returns generic types (application/octet-stream, text/plain) + # 2. Content detection fails or returns None + # 3. Extension detection provides a more specific type + + # Generic/unreliable MIME types that we should try to improve + generic_types = { + "application/octet-stream", + "application/x-ole-storage", # used by .xls, .doc and .ppt + "application/zip", + "text/plain", + } + + # If content detection gives us a generic type and we have extension info + if mimetype_from_content in generic_types and mimetype_from_extension: + # Use extension-based detection if it's more specific + if mimetype_from_extension not in generic_types: + return mimetype_from_extension + + # If content detection failed, returned None or is a generic type, use extension if available + if not mimetype_from_content or mimetype_from_content in generic_types: + if mimetype_from_extension: + return mimetype_from_extension + + # Default to content-based detection (most reliable) + return mimetype_from_content or "application/octet-stream" diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 3d370bf3..2dcc8be9 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -54,6 +54,7 @@ dependencies = [ "pydantic==2.12.4", "PyJWT==2.11.0", "python-frontmatter==1.1.0", + "python-magic==0.4.27", "requests==2.32.5", "sentry-sdk==2.53.0", "whitenoise==6.11.0",