gsadmin/meet
1
0
Fork 0
mirror of https://github.com/suitenumerique/meet.git synced 2026-07-26 11:58:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

✨(backend) add core.recording.event.parsers.S3Parser

Browse Source 
Implementation was validated against Ceph Object Store by @agasurfer.
Event payload matches the one from AWS S3 docs.
This commit is contained in:
ilias
2026-04-22 10:31:38 +02:00
committed by aleb_the_flash
parent 7d9f282c2e
commit 04dfb9922f
3 changed files with 221 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -15,6 +15,7 @@ and this project adheres to
- ✨(frontend) make reaction toolbar responsive on small viewports
- ✨(frontend) enable reactions on mobile devices
- ✨(frontend) introduce picture-in-picture meeting
- ✨(backend) add core.recording.event.parsers.S3Parser
### Changed
src/backend/core/recording/event/parsers.py
+56 -31
View File
@@ -1,6 +1,7 @@
"""Meet storage event parser classes."""
import logging
import mimetypes
import re
from dataclasses import dataclass
from functools import lru_cache
@@ -18,6 +19,9 @@ from .exceptions import (
ParsingEventDataError,
)
# Additional MIME type mapping
mimetypes.add_type("audio/ogg", ".ogg")
logger = logging.getLogger(__name__)
@@ -74,8 +78,8 @@ def get_parser() -> EventParser:
return event_parser_cls(bucket_name=settings.AWS_STORAGE_BUCKET_NAME)
class MinioParser:
"""Handle parsing and validation of Minio storage events."""
class BaseS3Parser:
"""Base class for handling parsing and validation of S3-compatible storage events."""
def __init__(self, bucket_name: str, allowed_filetypes=None):
"""Initialize parser with target bucket name and accepted filetypes."""
@@ -91,32 +95,6 @@ class MinioParser:
rf"(?P<url_encoded_folder_path>(?:[^%]+%2F)+)?{settings.RECORDING_OUTPUT_FOLDER}%2F(?P<recording_id>{UUID_REGEX})\.(?P<extension>{FILE_EXT_REGEX})"
)
@staticmethod
def parse(data):
"""Convert raw Minio event dictionary to StorageEvent object."""
if not data:
raise ParsingEventDataError("Received empty data.")
try:
record = data["Records"][0]
s3 = record["s3"]
bucket_name = s3["bucket"]["name"]
file_object = s3["object"]
filepath = file_object["key"]
filetype = file_object["contentType"]
except (KeyError, IndexError) as e:
raise ParsingEventDataError(f"Missing or malformed key: {e}.") from e
try:
return StorageEvent(
filepath=filepath,
filetype=filetype,
bucket_name=bucket_name,
metadata=None,
)
except TypeError as e:
raise ParsingEventDataError(f"Missing essential data fields: {e}") from e
def validate(self, event_data: StorageEvent) -> str:
"""Verify StorageEvent matches bucket, filetype and filepath requirements."""
@@ -141,9 +119,56 @@ class MinioParser:
return recording_id
def get_recording_id(self, data):
"""Extract recording ID from Minio event through parsing and validation."""
"""Extract recording ID from S3 event through parsing and validation."""
event_data = self.parse(data)
recording_id = self.validate(event_data)
return self.validate(event_data)
return recording_id
def parse(self, data: Dict) -> StorageEvent:
"""To be implemented by subclasses."""
raise NotImplementedError("Subclasses must implement parse()")
class MinioParser(BaseS3Parser):
"""Minio specific event parsing."""
def parse(self, data: Dict) -> StorageEvent:
if not data:
raise ParsingEventDataError("Received empty data.")
try:
record = data["Records"][0]
s3 = record["s3"]
return StorageEvent(
filepath=s3["object"]["key"],
filetype=s3["object"]["contentType"], # Minio-specific field
bucket_name=s3["bucket"]["name"],
metadata=None,
)
except (KeyError, IndexError) as e:
raise ParsingEventDataError(f"Malformed Minio event: {e}") from e
except TypeError as e:
raise ParsingEventDataError(f"Missing essential data fields: {e}") from e
class S3Parser(BaseS3Parser):
"""AWS S3 specific event parsing."""
def parse(self, data: Dict) -> StorageEvent:
if not data:
raise ParsingEventDataError("Received empty data.")
try:
# AWS S3 structure can slightly differ from Minio implementation
record = data["Records"][0]
s3 = record["s3"]
filepath = s3["object"]["key"]
if not filepath:
raise ParsingEventDataError("Missing object key name")
filetype, _ = mimetypes.guess_type(filepath)
return StorageEvent(
filepath=filepath,
filetype=filetype,
bucket_name=s3["bucket"]["name"],
metadata=None,
)
except (KeyError, IndexError) as e:
raise ParsingEventDataError(f"Malformed S3 event: {e}") from e
src/backend/core/tests/recording/event/test_parsers.py
+164 -36
View File
@@ -18,10 +18,13 @@ from core.recording.event.exceptions import (
)
from core.recording.event.parsers import (
MinioParser,
S3Parser,
StorageEvent,
get_parser,
)
# MinioParser
@pytest.fixture
def valid_minio_event():
@@ -47,7 +50,7 @@ def minio_parser():
return MinioParser(bucket_name="test-bucket")
def test_parse_valid_event(minio_parser, valid_minio_event):
def test_minio_parse_valid_event(minio_parser, valid_minio_event):
"""Test parsing a valid Minio event."""
event = minio_parser.parse(valid_minio_event)
assert isinstance(event, StorageEvent)
@@ -57,13 +60,33 @@ def test_parse_valid_event(minio_parser, valid_minio_event):
assert event.metadata is None
def test_parse_empty_data(minio_parser):
def test_minio_parse_with_video_type(minio_parser):
"""Test parsing event with video file type."""
video_event = {
"Records": [
{
"s3": {
"bucket": {"name": "test-bucket"},
"object": {
"key": "46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",
"contentType": "video/mp4",
},
}
}
]
}
event = minio_parser.parse(video_event)
assert event.filetype == "video/mp4"
assert event.filepath.endswith(".mp4")
def test_minio_parse_empty_data(minio_parser):
"""Test parsing empty event data raises error."""
with pytest.raises(ParsingEventDataError, match="Received empty data."):
minio_parser.parse({})
def test_parse_missing_keys(minio_parser):
def test_minio_parse_missing_keys(minio_parser):
"""Test parsing event with missing key."""
invalid_minio_event = {
@@ -77,11 +100,11 @@ def test_parse_missing_keys(minio_parser):
]
}
with pytest.raises(ParsingEventDataError, match="Missing or malformed key"):
with pytest.raises(ParsingEventDataError, match="Malformed Minio event:"):
minio_parser.parse(invalid_minio_event)
def test_parse_none_key(minio_parser):
def test_minio_parse_none_key(minio_parser):
"""Test parsing event with None field."""
invalid_minio_event = {
@@ -102,7 +125,7 @@ def test_parse_none_key(minio_parser):
minio_parser.parse(invalid_minio_event)
def test_validate_invalid_bucket(minio_parser):
def test_minio_validate_invalid_bucket(minio_parser):
"""Test validation with wrong bucket name."""
event = StorageEvent(
filepath="recording%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",
@@ -114,7 +137,7 @@ def test_validate_invalid_bucket(minio_parser):
minio_parser.validate(event)
def test_validate_invalid_filetype(minio_parser):
def test_minio_validate_invalid_filetype(minio_parser):
"""Test validation with unsupported file type."""
event = StorageEvent(
filepath="recording%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.txt",
@@ -139,7 +162,7 @@ def test_validate_invalid_filetype(minio_parser):
"folder%2Fuploads%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg", # nested but no recordings/
],
)
def test_validate_invalid_filepath(invalid_filepath, minio_parser):
def test_minio_validate_invalid_filepath(invalid_filepath, minio_parser):
"""Test validation with malformed filepath."""
event = StorageEvent(
filepath=invalid_filepath,
@@ -151,7 +174,7 @@ def test_validate_invalid_filepath(invalid_filepath, minio_parser):
minio_parser.validate(event)
def test_validate_valid_event(minio_parser):
def test_minio_validate_valid_event(minio_parser):
"""Test validation with valid event data."""
event = StorageEvent(
filepath="recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",
@@ -163,13 +186,13 @@ def test_validate_valid_event(minio_parser):
assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"
def test_get_recording_id_success(minio_parser, valid_minio_event):
def test_minio_get_recording_id_success(minio_parser, valid_minio_event):
"""Test successful extraction of recording ID."""
recording_id = minio_parser.get_recording_id(valid_minio_event)
assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"
def test_validate_filepath_with_folder(minio_parser):
def test_minio_validate_filepath_with_folder(minio_parser):
"""Test validation of filepath with folder structure."""
event = StorageEvent(
filepath="parent_folder%2Frecordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",
@@ -181,41 +204,21 @@ def test_validate_filepath_with_folder(minio_parser):
assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"
def test_parse_with_video_type(minio_parser):
"""Test parsing event with video file type."""
video_event = {
"Records": [
{
"s3": {
"bucket": {"name": "test-bucket"},
"object": {
"key": "46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",
"contentType": "video/mp4",
},
}
}
]
}
event = minio_parser.parse(video_event)
assert event.filetype == "video/mp4"
assert event.filepath.endswith(".mp4")
def test_empty_allowed_filetypes():
def test_minio_empty_allowed_filetypes():
"""Test MinioParser with empty allowed_filetypes."""
empty_types = set()
parser = MinioParser(bucket_name="test-bucket", allowed_filetypes=empty_types)
assert parser._allowed_filetypes == {"audio/ogg", "video/mp4"}
def test_custom_allowed_filetypes():
def test_minio_custom_allowed_filetypes():
"""Test MinioParser with empty allowed_filetypes."""
custom_types = {"audio/mp3", "video/mov"}
parser = MinioParser(bucket_name="test-bucket", allowed_filetypes=custom_types)
assert parser._allowed_filetypes == {"audio/mp3", "video/mov"}
def test_validate_custom_filetypes():
def test_minio_validate_custom_filetypes():
"""Test validation of filepath with folder structure."""
parser = MinioParser(bucket_name="test-bucket", allowed_filetypes={"audio/mp3"})
@@ -229,18 +232,143 @@ def test_validate_custom_filetypes():
parser.validate(event)
def test_constructor_none_bucket():
def test_minio_constructor_none_bucket():
"""Test MinioParser constructor with None bucket name."""
with pytest.raises(ValueError, match="Bucket name cannot be None or empty"):
MinioParser(bucket_name=None)
def test_constructor_empty_bucket():
def test_minio_constructor_empty_bucket():
"""Test MinioParser constructor with empty bucket name."""
with pytest.raises(ValueError, match="Bucket name cannot be None or empty"):
MinioParser(bucket_name="")
# S3Parser
@pytest.fixture
def valid_s3_event():
"""Mock a valid S3 event."""
return {
"Records": [
{
"s3": {
"bucket": {"name": "test-bucket"},
"object": {
"key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",
},
}
}
]
}
@pytest.fixture
def s3_parser():
"""Mock an S3 parser."""
return S3Parser(bucket_name="test-bucket")
def test_s3_parse_valid_event(s3_parser, valid_s3_event):
"""Test parsing a valid S3 event."""
event = s3_parser.parse(valid_s3_event)
assert isinstance(event, StorageEvent)
assert event.filepath == "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg"
assert event.filetype == "audio/ogg"
assert event.bucket_name == "test-bucket"
assert event.metadata is None
def test_s3_parse_empty_data(s3_parser):
"""Test parsing empty S3 event data raises error."""
with pytest.raises(ParsingEventDataError, match="Received empty data."):
s3_parser.parse({})
def test_s3_parse_missing_keys(s3_parser):
"""Test parsing S3 event with missing key."""
invalid_s3_event = {
"Records": [
{
"s3": {
"bucket": {"name": "test-bucket"},
# Missing 'object' key
}
}
]
}
with pytest.raises(ParsingEventDataError, match="Malformed S3 event:"):
s3_parser.parse(invalid_s3_event)
def test_s3_parse_none_key(s3_parser):
"""Test parsing S3 event with None field."""
invalid_s3_event = {
"Records": [
{
"s3": {
"bucket": {"name": "test-bucket"},
"object": {
"key": None,
},
}
}
]
}
with pytest.raises(ParsingEventDataError, match="Missing object key name"):
s3_parser.parse(invalid_s3_event)
def test_s3_parse_with_video_type(s3_parser):
"""Test parsing S3 event with mp4 file extension."""
video_event = {
"Records": [
{
"s3": {
"bucket": {"name": "test-bucket"},
"object": {
"key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",
},
}
}
]
}
event = s3_parser.parse(video_event)
assert event.filetype == "video/mp4"
assert event.filepath.endswith(".mp4")
def test_s3_parse_unrecognized_extension(s3_parser):
"""Test parsing S3 event with unrecognized file extension."""
event_with_unknown_ext = {
"Records": [
{
"s3": {
"bucket": {"name": "test-bucket"},
"object": {
"key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.zzunknown999",
},
}
}
]
}
with pytest.raises(TypeError, match="filetype cannot be None"):
s3_parser.parse(event_with_unknown_ext)
def test_s3_get_recording_id_success(s3_parser, valid_s3_event):
"""Test successful extraction of recording ID from S3 event."""
recording_id = s3_parser.get_recording_id(valid_s3_event)
assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"
# get_parser
@pytest.fixture
def clear_lru_cache():
"""Fixture to clear the LRU cache between tests."""