mirror of
https://github.com/suitenumerique/meet.git
synced 2026-11-17 21:32:27 +00:00
Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7f304bd0a6 | |||
| d5b7ed268d | |||
| d0379c2bd9 | |||
| a806cbe34a | |||
| 6e515d9e7b | |||
| da185ae979 | |||
| 572ec12057 | |||
| 3d05aa002c | |||
| cae6c19da8 | |||
| 455daf1e8f | |||
| 73b182b34a | |||
| 6a1622cd50 | |||
| 56b5a34ec5 |
+11
-8
@@ -2,19 +2,22 @@
|
||||
# Review regularly and remove entries once Debian ships a fix.
|
||||
|
||||
# util-linux
|
||||
CVE-2026-76642
|
||||
CVE-2026-78408
|
||||
CVE-2026-78409
|
||||
CVE-2026-78410
|
||||
CVE-2026-76642 exp:2026-12-05
|
||||
CVE-2026-78408 exp:2026-12-05
|
||||
CVE-2026-78409 exp:2026-12-05
|
||||
CVE-2026-78410 exp:2026-12-05
|
||||
|
||||
# acl
|
||||
CVE-2026-54369
|
||||
CVE-2026-54369 exp:2026-12-05
|
||||
|
||||
# ncurses
|
||||
CVE-2025-69720
|
||||
CVE-2025-69720 exp:2026-12-05
|
||||
|
||||
# systemd
|
||||
CVE-2026-16742
|
||||
CVE-2026-16742 exp:2026-12-05
|
||||
|
||||
# perl-base (fix deferred by Debian)
|
||||
CVE-2026-9538
|
||||
CVE-2026-9538 exp:2026-12-05
|
||||
|
||||
# libsqlite3-0 (fixed in sid 3.54.1-1; trixie marked no-dsa / minor)
|
||||
CVE-2026-119373 exp:2026-12-05
|
||||
|
||||
+41
-5
@@ -8,16 +8,52 @@ and this project adheres to
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.37.2] - 2026-11-17
|
||||
|
||||
## [1.37.1] - 2026-11-17
|
||||
|
||||
### Added
|
||||
|
||||
- ✨(backend) add OIDC_ALLOW_DUPLICATE_EMAILS to let users share an email
|
||||
- ✨(backend) show the last login of users in the admin
|
||||
- ✨(backend) filter users sharing an email in the admin
|
||||
|
||||
### Changed
|
||||
|
||||
- ⬆️(summary) update ffmpeg to 8.1.3-r0
|
||||
- 💥(backend) refuse new users whose email is already used
|
||||
- ♻️(backend) match emails in a single normalized form
|
||||
- ⚡️(backend) index user emails for exact lookups
|
||||
|
||||
### Fixed
|
||||
|
||||
- 🔒️(summary) protect signed URLs in models, source logs and analytics metadata
|
||||
- 🔒️(summary) mask secret validation inputs and sanitize upstream and task errors
|
||||
- 🔒️(summary) scrub sensitive Sentry event and breadcrumb content while retaining
|
||||
redacted frame locals and application DEBUG diagnostics
|
||||
- 📌(summary) pin kombu to keep celery workers compatible with redis 5
|
||||
- 🐛(frontend) show a page when the connection ends unexpectedly
|
||||
- 🐛(frontend) show the transcript mic hint when every mic is muted
|
||||
- 🐛(frontend) show join errors instead of silently waiting in the lobby
|
||||
- 🐛(frontend) send users who cannot enter directly back to the lobby
|
||||
- 🔒️(backend) refuse unregistered rooms whose slug is a UUID
|
||||
- 🔒️(agents) upgrade python to 3.14.10 to fix CVE-2026-109518
|
||||
- 🔒️(docker) upgrade musl-utils to 1.2.6-r3 to fix CVE-2026-107891
|
||||
- ♿️(frontend) close the invite dialog with the Escape key #1916
|
||||
- 🔒️(docker) move frontend to nginx/nginx-unprivileged image
|
||||
- 🐛(frontend) accept meeting links with query params in join dialog
|
||||
- 🐛(backend) allow muting when everyone_can_mute is stored as null #1942
|
||||
- 🐛(frontend) apply the sans font token on every page
|
||||
- 🐛(frontend) promote the active speaker in the carousel layout
|
||||
- ♿️(frontend) show the focus ring on connection test detail cards
|
||||
- ♿️(frontend) use a heading for every connection test card title
|
||||
- ♿️(frontend) put the ProConnect link label inside the link
|
||||
|
||||
>>>>>>> a3b4aa9d (♻️(backend) match emails in a single normalized form)
|
||||
## [1.37.0] - 2026-11-01
|
||||
|
||||
### Changed
|
||||
|
||||
- ⬆️(summary) update ffmpeg to 8.1.3-r0
|
||||
- ✨(frontend) replace the Crisp chat with the La Suite support form
|
||||
- 🔒️(frontend) keep rooms out of search engine and AI indexes
|
||||
- ⬆️(backend) upgrade django-lasuite to v1.0.0
|
||||
- ⚡️(docker) build frontend assets natively for arm64 images
|
||||
|
||||
## [1.36.3] - 2026-10-23
|
||||
|
||||
|
||||
+2
-1
@@ -9,7 +9,8 @@ RUN python -m pip install --upgrade pip
|
||||
# Upgrade system packages to install security updates
|
||||
RUN apk update && \
|
||||
apk upgrade && \
|
||||
apk add --no-cache --upgrade 'ca-certificates>=20261020-r0' && \
|
||||
apk add --no-cache --upgrade 'ca-certificates>=20261020-r0' \
|
||||
'musl-utils>=1.2.6-r3' && \
|
||||
update-ca-certificates
|
||||
|
||||
# ---- Back-end builder image ----
|
||||
|
||||
+26
@@ -16,6 +16,32 @@ the following command inside your docker container:
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### User emails matched in a single normalized form
|
||||
|
||||
User emails are now stored stripped, with their ASCII letters lowercased, and compared exactly. A migration normalizes
|
||||
the stored emails and stores blank ones as NULL. Other letters keep their case.
|
||||
|
||||
- Users now see their email in this form rather than as the identity provider sent it, and the API returns it the same
|
||||
way. An application comparing an email Meet returns with the one it sent, such as the `scope` of its application token request,
|
||||
must not compare them case-sensitively: it should strip both and lowercase their ASCII letters first.
|
||||
- With `OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION` disabled (the default), a new `sub` whose email is already used by
|
||||
another user is now refused, by the OIDC login and by external API calls made with a Menshen token, instead of getting
|
||||
a second account. To keep creating users with a duplicate email, set the new `OIDC_ALLOW_DUPLICATE_EMAILS=True`.
|
||||
|
||||
## v1.37.0
|
||||
|
||||
### Crisp support chat replaced by the La Suite support form
|
||||
|
||||
The Crisp chat is gone, the support button, now open the La Suite support form. They are hidden unless these `FRONTEND_SUPPORT` keys are all set.
|
||||
The former help links for recording/transcription are now redirecting to the adhoc page in the official documentation
|
||||
|
||||
### Pages hidden from search engines
|
||||
|
||||
Every page except `/` now carries a `noindex` tag and, with the provided nginx configurations, an `X-Robots-Tag` header to stay out of search indexes.
|
||||
A new `robots.txt` also blocks the main AI crawlers. If you maintain your own nginx configuration, copy the `location = /` and `location = /home.html` blocks from `src/frontend/default.conf`, and the `X-Robots-Tag` header from its `location /`.
|
||||
|
||||
## [v1.35.0]
|
||||
|
||||
### Purging deleted rooms
|
||||
|
||||
Deleting a room now only soft deletes it: it is hidden but keeps its slug and PIN code. The `purge_inactive_rooms` command also permanently deletes soft-deleted rooms. See [the room purge documentation](docs/features/room-purge.md).
|
||||
|
||||
@@ -38,6 +38,17 @@ server {
|
||||
add_header Cache-Control "public, max-age=2592000";
|
||||
}
|
||||
|
||||
location = / {
|
||||
try_files /home.html /index.html;
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache";
|
||||
add_header Expires 0;
|
||||
}
|
||||
|
||||
location = /home.html {
|
||||
internal;
|
||||
}
|
||||
|
||||
# Serve static files
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
@@ -45,6 +56,7 @@ server {
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache"; # HTTP 1.0 header for backward compatibility
|
||||
add_header Expires 0;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
}
|
||||
|
||||
# Optionally, handle 404 errors by redirecting to index.html
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# ---- Front-end image ----
|
||||
FROM node:22.23.4-alpine AS frontend-deps
|
||||
FROM --platform=$BUILDPLATFORM node:22.23.4-alpine AS frontend-deps
|
||||
|
||||
RUN apk upgrade --no-cache ca-certificates
|
||||
|
||||
@@ -41,7 +41,7 @@ COPY ./docker/dinum-frontend/fonts/ \
|
||||
./dist/assets/fonts/
|
||||
|
||||
# ---- Addons builder image ----
|
||||
FROM node:22.23.4-alpine AS addons-builder
|
||||
FROM --platform=$BUILDPLATFORM node:22.23.4-alpine AS addons-builder
|
||||
|
||||
RUN apk upgrade --no-cache ca-certificates
|
||||
|
||||
@@ -64,8 +64,9 @@ USER root
|
||||
RUN apk upgrade --no-cache libexpat && \
|
||||
apk add --no-cache --upgrade 'pcre2>=10.49-r0' 'tiff>=4.7.2-r0' \
|
||||
'libx11>=1.8.14-r0' 'ca-certificates>=20261020-r0' \
|
||||
'brotli-libs>=1.2.1-r0' && \
|
||||
apk del curl
|
||||
'brotli-libs>=1.2.1-r0' 'musl-utils>=1.2.6-r3' && \
|
||||
apk del curl && \
|
||||
rm -f /usr/share/nginx/html/50x.html
|
||||
USER nginx
|
||||
|
||||
USER nginx
|
||||
|
||||
@@ -43,6 +43,7 @@ server {
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache" always;
|
||||
add_header Expires 0 always;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
set $ms_domains "https://*.live.com https://*.office.com https://*.microsoft.com https://*.office365.com https://*.sharepoint.com";
|
||||
|
||||
@@ -76,6 +77,17 @@ server {
|
||||
add_header Cache-Control "public, max-age=2592000";
|
||||
}
|
||||
|
||||
location = / {
|
||||
try_files /home.html /index.html;
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache";
|
||||
add_header Expires 0;
|
||||
}
|
||||
|
||||
location = /home.html {
|
||||
internal;
|
||||
}
|
||||
|
||||
# Serve static files
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
@@ -83,6 +95,7 @@ server {
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache"; # HTTP 1.0 header for backward compatibility
|
||||
add_header Expires 0;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
}
|
||||
|
||||
# Optionally, handle 404 errors by redirecting to index.html
|
||||
|
||||
@@ -37,7 +37,7 @@ $ make help
|
||||
---
|
||||
|
||||
## Need Help?
|
||||
If you need any assistance or have questions while getting started, feel free to reach out to @lebaudantoine anytime! Antoine is available to help you onboard and guide you through the process. Chat with him @antoine.lebaud:matrix.org, or from the [support hotline](https://go.crisp.chat/chat/embed/?website_id=58ea6697-8eba-4492-bc59-ad6562585041).
|
||||
If you need any assistance or have questions while getting started, feel free to reach out to @lebaudantoine anytime! Antoine is available to help you onboard and guide you through the process. Chat with him @antoine.lebaud:matrix.org, or by email at [support-visio@numerique.gouv.fr](mailto:support-visio@numerique.gouv.fr).
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -348,7 +348,7 @@ These are the environmental options available on meet backend.
|
||||
| CSRF_TRUSTED_ORIGINS | CSRF trusted origins list | [] |
|
||||
| FRONTEND_CUSTOM_CSS_URL | URL of an additional CSS file to load in the frontend app. If set, a `<link>` tag with this URL as href is added to the `<head>` of the frontend app | |
|
||||
| FRONTEND_ANALYTICS | Analytics information | {} |
|
||||
| FRONTEND_SUPPORT | Crisp frontend support configuration, also you can pass help articles, with `help_article_transcript`, `help_article_recording`, `help_article_more_tools` | {} |
|
||||
| FRONTEND_SUPPORT | La Suite support form (`widget_script_url`, `widget_api_url`, `widget_channel`) and help articles (`help_article_*`) | {} |
|
||||
| FRONTEND_MANIFEST_LINK | Link to the "Learn more" button on the homepage | {} |
|
||||
| FRONTEND_SILENCE_LIVEKIT_DEBUG | Silence LiveKit debug logs | false |
|
||||
| FRONTEND_IS_SILENT_LOGIN_ENABLED | Enable silent login feature | true |
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
<html lang="fr">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="robots" content="noindex, nofollow" />
|
||||
<title data-app-name></title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
|
||||
<script nonce="NONCE_PLACEHOLDER" src="https://appsforoffice.microsoft.com/lib/1/hosted/office.js"></script>
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="robots" content="noindex, nofollow" />
|
||||
<title data-app-name></title>
|
||||
<link rel="stylesheet" href="../styles/spinner.css" />
|
||||
<script nonce="NONCE_PLACEHOLDER" src="/addons/outlook/config.js"></script>
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="robots" content="noindex, nofollow" />
|
||||
<title data-app-name></title>
|
||||
<link rel="stylesheet" href="taskpane.css" />
|
||||
<script nonce="NONCE_PLACEHOLDER" src="/addons/outlook/config.js"></script>
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="robots" content="noindex, nofollow" />
|
||||
<title data-app-name></title>
|
||||
<link rel="stylesheet" href="../styles/spinner.css" />
|
||||
<script nonce="NONCE_PLACEHOLDER" src="/addons/outlook/config.js"></script>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
[project]
|
||||
name = "agents"
|
||||
version = "1.36.3"
|
||||
version = "1.37.2"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"livekit-agents==1.7.0",
|
||||
|
||||
Generated
+1
-1
@@ -9,7 +9,7 @@ resolution-markers = [
|
||||
|
||||
[[package]]
|
||||
name = "agents"
|
||||
version = "1.36.3"
|
||||
version = "1.37.2"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "boto3" },
|
||||
|
||||
@@ -4,6 +4,7 @@ from django import forms
|
||||
from django.contrib import admin, messages
|
||||
from django.contrib.auth import admin as auth_admin
|
||||
from django.db import transaction
|
||||
from django.db.models import Exists, OuterRef
|
||||
from django.utils.html import format_html
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
@@ -46,6 +47,26 @@ class FileInline(admin.TabularInline):
|
||||
return super().get_queryset(request).filter(hard_deleted_at__isnull=True)
|
||||
|
||||
|
||||
class DuplicateEmailFilter(admin.SimpleListFilter):
|
||||
"""Filter the users whose stored email is shared with another user."""
|
||||
|
||||
title = _("duplicate email")
|
||||
parameter_name = "duplicate_email"
|
||||
|
||||
def lookups(self, request, model_admin):
|
||||
"""Return the filter options."""
|
||||
return [("yes", _("Yes"))]
|
||||
|
||||
def queryset(self, request, queryset):
|
||||
"""Keep the users sharing their email with another user."""
|
||||
if self.value() != "yes":
|
||||
return queryset
|
||||
others = models.User.objects.filter(email=OuterRef("email")).exclude(
|
||||
pk=OuterRef("pk")
|
||||
)
|
||||
return queryset.filter(Exists(others))
|
||||
|
||||
|
||||
@admin.register(models.User)
|
||||
class UserAdmin(auth_admin.UserAdmin):
|
||||
"""Admin class for the User model"""
|
||||
@@ -89,7 +110,10 @@ class UserAdmin(auth_admin.UserAdmin):
|
||||
),
|
||||
},
|
||||
),
|
||||
(_("Important dates"), {"fields": ("created_at", "updated_at")}),
|
||||
(
|
||||
_("Important dates"),
|
||||
{"fields": ("created_at", "updated_at", "last_login")},
|
||||
),
|
||||
)
|
||||
add_fieldsets = (
|
||||
(
|
||||
@@ -112,9 +136,16 @@ class UserAdmin(auth_admin.UserAdmin):
|
||||
"is_superuser",
|
||||
"is_device",
|
||||
"created_at",
|
||||
"last_login",
|
||||
"updated_at",
|
||||
)
|
||||
list_filter = ("is_staff", "is_superuser", "is_device", "is_active")
|
||||
list_filter = (
|
||||
"is_staff",
|
||||
"is_superuser",
|
||||
"is_device",
|
||||
"is_active",
|
||||
DuplicateEmailFilter,
|
||||
)
|
||||
ordering = (
|
||||
"is_active",
|
||||
"-is_superuser",
|
||||
@@ -131,9 +162,16 @@ class UserAdmin(auth_admin.UserAdmin):
|
||||
"short_name",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"last_login",
|
||||
)
|
||||
search_fields = ("id", "sub", "admin_email", "email", "full_name")
|
||||
|
||||
def get_ordering(self, request):
|
||||
"""Group the users sharing an email when filtering on duplicates."""
|
||||
if DuplicateEmailFilter.parameter_name in request.GET:
|
||||
return ("email", "created_at")
|
||||
return super().get_ordering(request)
|
||||
|
||||
|
||||
@admin.register(models.File)
|
||||
class FileAdmin(admin.ModelAdmin):
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Authentication Backends for the Meet core app."""
|
||||
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import (
|
||||
SuspiciousOperation,
|
||||
@@ -13,9 +15,13 @@ from lasuite.oidc_login.backends import (
|
||||
)
|
||||
from rest_framework.authentication import SessionAuthentication
|
||||
|
||||
from core.models import User
|
||||
from core import audit, auditing
|
||||
from core.models import DuplicateEmailError, User
|
||||
from core.utils import normalize_email
|
||||
from core.validators import sub_validator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OIDCAuthenticationBackend(LaSuiteOIDCAuthenticationBackend):
|
||||
"""Custom OpenID Connect (OIDC) Authentication Backend.
|
||||
@@ -83,18 +89,25 @@ class OIDCAuthenticationBackend(LaSuiteOIDCAuthenticationBackend):
|
||||
) from err
|
||||
|
||||
try:
|
||||
return User.objects.get(sub=sub)
|
||||
except User.DoesNotExist:
|
||||
if email and settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION:
|
||||
try:
|
||||
return User.objects.get(email__iexact=email)
|
||||
except User.DoesNotExist:
|
||||
pass
|
||||
except User.MultipleObjectsReturned as e:
|
||||
raise SuspiciousOperation(
|
||||
"Multiple user accounts share a common email."
|
||||
) from e
|
||||
return None
|
||||
return User.objects.get_user_by_sub_or_email(sub, email)
|
||||
except DuplicateEmailError as err:
|
||||
logger.warning("Email already used, not creating user: %s", sub)
|
||||
existing_user = (
|
||||
User.objects.filter(email=normalize_email(email))
|
||||
.order_by("-is_active", "-created_at")
|
||||
.first()
|
||||
)
|
||||
audit.log(
|
||||
auditing.USER_PROVISION,
|
||||
outcome=audit.Outcome.FAILURE,
|
||||
reason=audit.Reason.CONFLICT,
|
||||
actor=None,
|
||||
actor_type=audit.ActorType.ANONYMOUS,
|
||||
auth_method="oidc",
|
||||
target=existing_user,
|
||||
requested_sub=sub,
|
||||
)
|
||||
raise SuspiciousOperation(err.message) from err
|
||||
|
||||
|
||||
class SessionAuthenticationWith401(SessionAuthentication):
|
||||
|
||||
@@ -20,6 +20,7 @@ from rest_framework import authentication, exceptions
|
||||
from core import audit, auditing
|
||||
from core.models import Application, ApplicationScope
|
||||
from core.services import jwt_token
|
||||
from core.utils import normalize_email
|
||||
from core.validators import sub_validator
|
||||
|
||||
User = get_user_model()
|
||||
@@ -63,6 +64,10 @@ def get_or_create_user_by_sub(sub, email=None, *, auth_method, client_id=None):
|
||||
Shared by the resource server backends, so that tokens introspected with the
|
||||
OIDC provider or with Menshen resolve users the same way.
|
||||
|
||||
Users are never identified by email here, but a user to create is refused
|
||||
when its normalized email is already stored, unless
|
||||
OIDC_ALLOW_DUPLICATE_EMAILS is set.
|
||||
|
||||
Args:
|
||||
sub: Subject identifier from the introspected token
|
||||
email: Email to store on a newly created user
|
||||
@@ -90,12 +95,10 @@ def get_or_create_user_by_sub(sub, email=None, *, auth_method, client_id=None):
|
||||
if not settings.OIDC_CREATE_USER:
|
||||
return None
|
||||
|
||||
# Users are never identified by email here. But a second account with the
|
||||
# same email (e.g. provisionnal account) would break the lookups by email:
|
||||
# refuse it.
|
||||
normalized_email = normalize_email(email)
|
||||
existing_user = (
|
||||
User.objects.filter(email__iexact=email).first()
|
||||
if settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION and email
|
||||
User.objects.filter(email=normalized_email).first()
|
||||
if normalized_email and not settings.OIDC_ALLOW_DUPLICATE_EMAILS
|
||||
else None
|
||||
)
|
||||
if existing_user is not None:
|
||||
|
||||
@@ -6,7 +6,6 @@ from django.contrib.auth import get_user_model
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db import transaction
|
||||
from django.db.models import Count
|
||||
from django.db.models.functions import Lower
|
||||
|
||||
from core.models import File, RecordingAccess, ResourceAccess, RoleChoices
|
||||
|
||||
@@ -21,14 +20,15 @@ ROLE_PRIORITY = {
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""
|
||||
Merge duplicate users sharing the same email (case-insensitive) into the
|
||||
most recently created one.
|
||||
Merge duplicate users sharing the same email into the most recently
|
||||
created one.
|
||||
|
||||
Emails are compared case-insensitively, so 'John@Example.com' and
|
||||
'john@example.com' are treated as duplicates. The KEPT user is the most
|
||||
recently created. All room memberships, recording accesses and files are
|
||||
transferred to it. When a conflict exists, the higher-privilege role wins.
|
||||
Stale users are then deleted.
|
||||
Emails are stored normalized (stripped, ASCII letters lowercased) and are
|
||||
compared exactly, so 'John@Example.com' and 'john@example.com' are both
|
||||
stored as 'john@example.com' and treated as duplicates. The KEPT user is
|
||||
the most recently created. All room memberships, recording accesses and
|
||||
files are transferred to it. When a conflict exists, the higher-privilege
|
||||
role wins. Stale users are then deleted.
|
||||
Each email group is processed inside a single database transaction.
|
||||
"""
|
||||
|
||||
@@ -60,16 +60,13 @@ class Command(BaseCommand):
|
||||
users_qs = users_qs.filter(email__icontains=email_filter)
|
||||
self.stdout.write(f"[INFO] Filtering emails containing '{email_filter}'.\n")
|
||||
|
||||
# Group emails case-insensitively so 'John@X.com' and 'john@x.com'
|
||||
# are detected as duplicates of each other.
|
||||
duplicate_emails = (
|
||||
users_qs.exclude(email__isnull=True)
|
||||
.exclude(email="")
|
||||
.annotate(email_lower=Lower("email"))
|
||||
.values("email_lower")
|
||||
.values("email")
|
||||
.annotate(cnt=Count("id"))
|
||||
.filter(cnt__gt=1)
|
||||
.values_list("email_lower", flat=True)
|
||||
.values_list("email", flat=True)
|
||||
)
|
||||
|
||||
if not duplicate_emails:
|
||||
@@ -85,12 +82,9 @@ class Command(BaseCommand):
|
||||
failed_emails = []
|
||||
|
||||
for email in duplicate_emails:
|
||||
# Case-insensitive lookup to fetch every casing variant of the email.
|
||||
# Secondary sort by id ensures a stable, deterministic order when
|
||||
# created_at timestamps are equal (common in tests and bulk imports).
|
||||
users = list(
|
||||
User.objects.filter(email__iexact=email).order_by("created_at", "id")
|
||||
)
|
||||
users = list(User.objects.filter(email=email).order_by("created_at", "id"))
|
||||
kept_user = users[-1]
|
||||
stale_users = users[:-1]
|
||||
|
||||
@@ -130,10 +124,6 @@ class Command(BaseCommand):
|
||||
failed_emails.append(email)
|
||||
self.stderr.write(f"[ERROR] Failed to merge '{email}': {exc}")
|
||||
|
||||
if not kept_user.email.islower():
|
||||
kept_user.email = kept_user.email.lower()
|
||||
kept_user.save(update_fields=["email"])
|
||||
|
||||
if failed_emails:
|
||||
raise CommandError(
|
||||
f"Failed to merge {len(failed_emails)} email group(s): {', '.join(failed_emails)}"
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Store user emails stripped and ASCII-lowercased, and blank ones as NULL."""
|
||||
|
||||
import string
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
BATCH_SIZE = 500
|
||||
|
||||
_ASCII_LOWERCASE_TABLE = str.maketrans(string.ascii_uppercase, string.ascii_lowercase)
|
||||
|
||||
|
||||
def normalize_email(email):
|
||||
"""Strip an email and lowercase its ASCII letters only."""
|
||||
return (email or "").strip().translate(_ASCII_LOWERCASE_TABLE)
|
||||
|
||||
|
||||
def forwards(apps, schema_editor):
|
||||
"""Normalize the user emails, writing the changed rows in batches."""
|
||||
User = apps.get_model("core", "User")
|
||||
batch = []
|
||||
for user_id, email in User.objects.values_list("id", "email").iterator(
|
||||
chunk_size=BATCH_SIZE
|
||||
):
|
||||
normalized = normalize_email(email) or None
|
||||
if normalized == email:
|
||||
continue
|
||||
batch.append(User(id=user_id, email=normalized))
|
||||
if len(batch) == BATCH_SIZE:
|
||||
User.objects.bulk_update(batch, ["email"])
|
||||
batch = []
|
||||
if batch:
|
||||
User.objects.bulk_update(batch, ["email"])
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
atomic = False
|
||||
|
||||
dependencies = [
|
||||
("core", "0029_alter_application_client_secret_sha256"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(forwards, migrations.RunPython.noop, elidable=True),
|
||||
]
|
||||
@@ -0,0 +1,20 @@
|
||||
# Generated by Django 5.2.17 on 2026-11-10 16:28
|
||||
|
||||
import core.models
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0030_normalize_user_emails'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterModelManagers(
|
||||
name='user',
|
||||
managers=[
|
||||
('objects', core.models.UserManager()),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,26 @@
|
||||
# Generated by Django 5.2.17 on 2026-11-10 17:00
|
||||
|
||||
from django.contrib.postgres.operations import (
|
||||
AddIndexConcurrently,
|
||||
RemoveIndexConcurrently,
|
||||
)
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
atomic = False
|
||||
|
||||
dependencies = [
|
||||
('core', '0031_alter_user_managers'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
AddIndexConcurrently(
|
||||
model_name="user",
|
||||
index=models.Index(fields=["email"], name="user_email_idx"),
|
||||
),
|
||||
RemoveIndexConcurrently(
|
||||
model_name="user",
|
||||
name="user_email_upper_idx",
|
||||
),
|
||||
]
|
||||
@@ -115,6 +115,16 @@ class RoomAccessLevel(models.TextChoices):
|
||||
RESTRICTED = "restricted", _("Restricted Access")
|
||||
|
||||
|
||||
class DuplicateEmailError(Exception):
|
||||
"""Raised when an email is already associated with a pre-existing user."""
|
||||
|
||||
def __init__(self, message=None, email=None):
|
||||
"""Set message and email to describe the exception."""
|
||||
self.message = message
|
||||
self.email = email
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class BaseModel(models.Model):
|
||||
"""
|
||||
Serves as an abstract base model for other models, ensuring that records are validated
|
||||
@@ -152,6 +162,36 @@ class BaseModel(models.Model):
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
|
||||
class UserManager(auth_models.UserManager):
|
||||
"""Custom manager for User model with additional methods."""
|
||||
|
||||
def get_user_by_sub_or_email(self, sub, email):
|
||||
"""Fetch existing user by sub or email."""
|
||||
try:
|
||||
return self.get(sub=sub)
|
||||
except self.model.DoesNotExist as err:
|
||||
normalized_email = utils.normalize_email(email)
|
||||
if not normalized_email:
|
||||
return None
|
||||
|
||||
if settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION:
|
||||
users = self.filter(email=normalized_email).order_by("-is_active")[:2]
|
||||
if len(users) == 2 and users[1].is_active:
|
||||
raise DuplicateEmailError(
|
||||
"Several active users are associated with this email."
|
||||
) from err
|
||||
return users[0] if users else None
|
||||
if (
|
||||
not settings.OIDC_ALLOW_DUPLICATE_EMAILS
|
||||
and self.filter(email=normalized_email).exists()
|
||||
):
|
||||
raise DuplicateEmailError(
|
||||
"We couldn't find a user with this sub but the email is already "
|
||||
"associated with a registered user."
|
||||
) from err
|
||||
return None
|
||||
|
||||
|
||||
class User(AbstractBaseUser, BaseModel, auth_models.PermissionsMixin):
|
||||
"""User model to work with OIDC only authentication."""
|
||||
|
||||
@@ -229,7 +269,7 @@ class User(AbstractBaseUser, BaseModel, auth_models.PermissionsMixin):
|
||||
),
|
||||
)
|
||||
|
||||
objects = auth_models.UserManager()
|
||||
objects = UserManager()
|
||||
|
||||
USERNAME_FIELD = "admin_email"
|
||||
REQUIRED_FIELDS = []
|
||||
@@ -247,12 +287,21 @@ class User(AbstractBaseUser, BaseModel, auth_models.PermissionsMixin):
|
||||
)
|
||||
]
|
||||
indexes = [
|
||||
models.Index(models.functions.Upper("email"), name="user_email_upper_idx"),
|
||||
models.Index(fields=["email"], name="user_email_idx"),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return self.email or self.admin_email or str(self.id)
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
"""Store the email normalized, and a blank one as NULL, even on a partial save."""
|
||||
email = utils.normalize_email(self.email) or None
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if update_fields is not None and email != self.email:
|
||||
kwargs["update_fields"] = {*update_fields, "email"}
|
||||
self.email = email
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
def email_user(self, subject, message, from_email=None, **kwargs):
|
||||
"""Email this user."""
|
||||
if not self.email:
|
||||
|
||||
@@ -5,6 +5,7 @@ from django.core.exceptions import SuspiciousOperation, ValidationError
|
||||
from django.db import IntegrityError
|
||||
|
||||
from core import audit, auditing, models
|
||||
from core.utils import normalize_email
|
||||
|
||||
|
||||
class ProvisionalUserError(Exception):
|
||||
@@ -42,7 +43,7 @@ class ProvisionalUserService:
|
||||
def _get_by_email(self, email: str) -> models.User | None:
|
||||
"""Return the user with this email, or None if not found."""
|
||||
try:
|
||||
return models.User.objects.get(email__iexact=email)
|
||||
return models.User.objects.get(email=normalize_email(email))
|
||||
except models.User.DoesNotExist:
|
||||
return None
|
||||
except models.User.MultipleObjectsReturned as e:
|
||||
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
from lasuite.marketing.tasks import create_or_update_contact
|
||||
|
||||
from core import models
|
||||
from core.audit.testing import find_events
|
||||
from core.authentication.backends import OIDCAuthenticationBackend
|
||||
from core.factories import UserFactory
|
||||
|
||||
@@ -296,17 +297,26 @@ def test_finds_user_by_sub(django_assert_num_queries):
|
||||
|
||||
|
||||
def test_finds_user_when_email_fallback_disabled(django_assert_num_queries, settings):
|
||||
"""Should not return a user when not found by sub, and email fallback is disabled."""
|
||||
"""Should refuse a sub that is not found when its email is already used, and email
|
||||
fallback is disabled."""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = False
|
||||
settings.OIDC_ALLOW_DUPLICATE_EMAILS = False
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
db_user = UserFactory(email="foo@mail.com")
|
||||
|
||||
with django_assert_num_queries(1):
|
||||
user = klass.get_existing_user("wrong-sub", db_user.email)
|
||||
|
||||
assert user is None
|
||||
with (
|
||||
django_assert_num_queries(3),
|
||||
pytest.raises(
|
||||
SuspiciousOperation,
|
||||
match=(
|
||||
"We couldn't find a user with this sub but the email is already "
|
||||
"associated with a registered user."
|
||||
),
|
||||
),
|
||||
):
|
||||
klass.get_existing_user("wrong-sub", db_user.email)
|
||||
|
||||
|
||||
def test_finds_user_when_email_is_none(django_assert_num_queries, settings):
|
||||
@@ -340,7 +350,7 @@ def test_finds_user_by_email(django_assert_num_queries, settings):
|
||||
|
||||
|
||||
def test_finds_user_case_insensitive_email(django_assert_num_queries, settings):
|
||||
"""Should match email case-insensitively when falling back to email."""
|
||||
"""Should match email ignoring ASCII case when falling back to email."""
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = True
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
@@ -353,7 +363,7 @@ def test_finds_user_case_insensitive_email(django_assert_num_queries, settings):
|
||||
|
||||
|
||||
def test_finds_user_multiple_users_same_email(django_assert_num_queries, settings):
|
||||
"""Should handle multiple users with same email appropriately."""
|
||||
"""Should refuse to pick one of several active users sharing the email."""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = True
|
||||
|
||||
@@ -363,30 +373,278 @@ def test_finds_user_multiple_users_same_email(django_assert_num_queries, setting
|
||||
UserFactory(email=email) # Second user with same email
|
||||
|
||||
with (
|
||||
django_assert_num_queries(2),
|
||||
django_assert_num_queries(3),
|
||||
pytest.raises(
|
||||
SuspiciousOperation,
|
||||
match="Multiple user accounts share a common email.",
|
||||
match="Several active users are associated with this email.",
|
||||
),
|
||||
):
|
||||
klass.get_existing_user("wrong-sub", email)
|
||||
|
||||
|
||||
def test_finds_user_whitespace_email(django_assert_num_queries, settings):
|
||||
"""Should not match emails with whitespace."""
|
||||
"""Should match emails with surrounding whitespace once stripped."""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = True
|
||||
settings.OIDC_CREATE_USER = False
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
UserFactory(email="foo@mail.com")
|
||||
db_user = UserFactory(email="foo@mail.com")
|
||||
|
||||
with django_assert_num_queries(2):
|
||||
user = klass.get_existing_user("wrong-sub", " foo@mail.com ")
|
||||
|
||||
assert user == db_user
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fallback", [True, False])
|
||||
def test_finds_user_whitespace_only_email(
|
||||
django_assert_num_queries, settings, fallback
|
||||
):
|
||||
"""Should not look a user up by email when the email is empty once stripped, even
|
||||
if a legacy user is stored with an empty email."""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = fallback
|
||||
settings.OIDC_ALLOW_DUPLICATE_EMAILS = False
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
legacy_user = UserFactory()
|
||||
models.User.objects.filter(pk=legacy_user.pk).update(email="")
|
||||
|
||||
with django_assert_num_queries(1):
|
||||
user = klass.get_existing_user("wrong-sub", " ")
|
||||
|
||||
assert user is None
|
||||
|
||||
|
||||
def test_authentication_getter_existing_user_via_email_case_insensitive(
|
||||
monkeypatch, settings
|
||||
):
|
||||
"""
|
||||
If an existing user doesn't match the sub but matches the email with another
|
||||
ASCII case, the user should be returned and its email should stay normalized.
|
||||
"""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = True
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
db_user = UserFactory(email="john.doe@example.com")
|
||||
|
||||
def get_userinfo_mocked(*args):
|
||||
return {"sub": "123", "email": "JOHN.DOE@EXAMPLE.COM"}
|
||||
|
||||
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
|
||||
|
||||
user = klass.get_or_create_user(
|
||||
access_token="test-token", id_token=None, payload=None
|
||||
)
|
||||
|
||||
assert user == db_user
|
||||
db_user.refresh_from_db()
|
||||
assert db_user.email == "john.doe@example.com"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"email",
|
||||
[
|
||||
"dısk@example.com",
|
||||
"diſk@example.com",
|
||||
"disK@example.com",
|
||||
],
|
||||
ids=["dotless-i", "long-s", "kelvin-sign"],
|
||||
)
|
||||
@pytest.mark.parametrize("fallback", [True, False])
|
||||
def test_authentication_getter_existing_user_via_email_non_ascii(
|
||||
monkeypatch, settings, fallback, email
|
||||
):
|
||||
"""
|
||||
If an existing user doesn't match the sub and the email only matches its email
|
||||
through Unicode case folding, a new user should be created and the existing user
|
||||
should be left untouched.
|
||||
"""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = fallback
|
||||
settings.OIDC_ALLOW_DUPLICATE_EMAILS = False
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
db_user = UserFactory(email="disk@example.com")
|
||||
db_sub = db_user.sub
|
||||
|
||||
def get_userinfo_mocked(*args):
|
||||
return {"sub": "other-sub", "email": email}
|
||||
|
||||
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
|
||||
|
||||
user = klass.get_or_create_user(
|
||||
access_token="test-token", id_token=None, payload=None
|
||||
)
|
||||
|
||||
assert user != db_user
|
||||
assert user.sub == "other-sub"
|
||||
assert user.email == email
|
||||
assert models.User.objects.count() == 2
|
||||
db_user.refresh_from_db()
|
||||
assert db_user.sub == db_sub
|
||||
assert db_user.email == "disk@example.com"
|
||||
|
||||
|
||||
def test_authentication_getter_existing_inactive_user_via_email(monkeypatch, settings):
|
||||
"""
|
||||
If the only user matching the email is inactive, the login should be refused
|
||||
instead of creating another user.
|
||||
"""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = True
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
UserFactory(email="john.doe@example.com", is_active=False)
|
||||
|
||||
def get_userinfo_mocked(*args):
|
||||
return {"sub": "123", "email": "John.Doe@example.com"}
|
||||
|
||||
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
|
||||
|
||||
with pytest.raises(SuspiciousOperation, match="User account is disabled"):
|
||||
klass.get_or_create_user(access_token="test-token", id_token=None, payload=None)
|
||||
|
||||
assert models.User.objects.count() == 1
|
||||
|
||||
|
||||
def test_authentication_getter_no_fallback_to_email_no_duplicate(monkeypatch, settings):
|
||||
"""
|
||||
When the email fallback is disabled, an unknown sub whose email is already used,
|
||||
in another ASCII case, should be refused instead of creating a second user.
|
||||
"""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = False
|
||||
settings.OIDC_ALLOW_DUPLICATE_EMAILS = False
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
db_user = UserFactory(email="john.doe@example.com")
|
||||
|
||||
def get_userinfo_mocked(*args):
|
||||
return {"sub": "123", "email": "JOHN.DOE@EXAMPLE.COM"}
|
||||
|
||||
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
|
||||
|
||||
with pytest.raises(
|
||||
SuspiciousOperation,
|
||||
match=(
|
||||
"We couldn't find a user with this sub but the email is already "
|
||||
"associated with a registered user."
|
||||
),
|
||||
):
|
||||
klass.get_or_create_user(access_token="test-token", id_token=None, payload=None)
|
||||
|
||||
assert models.User.objects.count() == 1
|
||||
db_user.refresh_from_db()
|
||||
assert db_user.sub != "123"
|
||||
|
||||
|
||||
def test_authentication_getter_no_fallback_to_email_no_duplicate_is_audited(
|
||||
audit_events, caplog, monkeypatch, settings
|
||||
):
|
||||
"""
|
||||
A login refused because its email is already used should log the refused sub, and
|
||||
be audited as a conflict with the user holding the email, never naming the email.
|
||||
"""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = False
|
||||
settings.OIDC_ALLOW_DUPLICATE_EMAILS = False
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
db_user = UserFactory(email="john.doe@example.com")
|
||||
|
||||
def get_userinfo_mocked(*args):
|
||||
return {"sub": "new-sub", "email": "JOHN.DOE@EXAMPLE.COM"}
|
||||
|
||||
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
|
||||
|
||||
with pytest.raises(SuspiciousOperation):
|
||||
klass.get_or_create_user(access_token="test-token", id_token=None, payload=None)
|
||||
|
||||
assert "Email already used, not creating user: new-sub" in caplog.text
|
||||
assert "john.doe@example.com" not in caplog.text.lower()
|
||||
|
||||
[provision] = find_events(audit_events, "user.provision")
|
||||
|
||||
assert provision["event"]["outcome"] == "failure"
|
||||
assert provision["event"]["reason"] == "conflict"
|
||||
assert provision["lasuite"]["actor"] == {"type": "anonymous"}
|
||||
assert provision["lasuite"]["auth"] == {"method": "oidc"}
|
||||
assert provision["user"] == {
|
||||
"target": {"id": str(db_user.pk), "domain": "example.com"}
|
||||
}
|
||||
assert provision["entity"]["target"]["id"] == str(db_user.pk)
|
||||
assert provision["entity"]["target"]["raw"] == {"sub": db_user.sub}
|
||||
assert provision["lasuite"]["details"] == {"requested_sub": "new-sub"}
|
||||
assert provision["log"]["level"] == "warning"
|
||||
assert "john.doe@example.com" not in str(provision).lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fallback", [True, False])
|
||||
def test_authentication_getter_email_conflict_audits_active_user(
|
||||
audit_events, fallback, monkeypatch, settings
|
||||
):
|
||||
"""
|
||||
A login refused because of its email, with or without the email fallback, should
|
||||
be audited against the newest active user holding the email, not a newer inactive
|
||||
one.
|
||||
"""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = fallback
|
||||
settings.OIDC_ALLOW_DUPLICATE_EMAILS = False
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
UserFactory(email="john.doe@example.com")
|
||||
active_user = UserFactory(email="john.doe@example.com")
|
||||
UserFactory(email="john.doe@example.com", is_active=False)
|
||||
|
||||
def get_userinfo_mocked(*args):
|
||||
return {"sub": "new-sub", "email": "john.doe@example.com"}
|
||||
|
||||
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
|
||||
|
||||
with pytest.raises(SuspiciousOperation):
|
||||
klass.get_or_create_user(access_token="test-token", id_token=None, payload=None)
|
||||
|
||||
[provision] = find_events(audit_events, "user.provision")
|
||||
|
||||
assert provision["event"]["reason"] == "conflict"
|
||||
assert provision["user"]["target"]["id"] == str(active_user.pk)
|
||||
assert provision["lasuite"]["details"] == {"requested_sub": "new-sub"}
|
||||
assert models.User.objects.count() == 3
|
||||
|
||||
|
||||
def test_authentication_getter_no_fallback_to_email_allow_duplicate(
|
||||
monkeypatch, settings
|
||||
):
|
||||
"""
|
||||
When the email fallback is disabled and duplicate emails are allowed, an unknown
|
||||
sub whose email is already used should get a new user.
|
||||
"""
|
||||
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = False
|
||||
settings.OIDC_ALLOW_DUPLICATE_EMAILS = True
|
||||
|
||||
klass = OIDCAuthenticationBackend()
|
||||
db_user = UserFactory(email="john.doe@example.com")
|
||||
|
||||
def get_userinfo_mocked(*args):
|
||||
return {"sub": "123", "email": "JOHN.DOE@EXAMPLE.COM"}
|
||||
|
||||
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
|
||||
|
||||
user = klass.get_or_create_user(
|
||||
access_token="test-token", id_token=None, payload=None
|
||||
)
|
||||
|
||||
assert user != db_user
|
||||
assert user.sub == "123"
|
||||
assert user.email == "john.doe@example.com"
|
||||
assert models.User.objects.count() == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"email",
|
||||
[
|
||||
|
||||
@@ -19,6 +19,13 @@ pytestmark = pytest.mark.django_db
|
||||
# pylint: disable=W0613
|
||||
|
||||
|
||||
def _user_with_stored_email(email):
|
||||
"""Create a user storing the email as is, like the rows predating the normalization."""
|
||||
user = UserFactory()
|
||||
User.objects.filter(pk=user.pk).update(email=email)
|
||||
return user
|
||||
|
||||
|
||||
def test_merge_no_duplicates_does_nothing():
|
||||
"""Command should do nothing when no duplicate users exist."""
|
||||
user = UserFactory(email="unique@example.com")
|
||||
@@ -36,24 +43,30 @@ def test_merge_keeps_most_recently_created_user():
|
||||
assert User.objects.filter(id=user2.id).exists()
|
||||
|
||||
|
||||
def test_merge_user_case_insensitive():
|
||||
"""Emails differing only by case should be treated as duplicates and merged,
|
||||
keeping the most recently created user."""
|
||||
user1 = UserFactory(email="Dup@example.com")
|
||||
user2 = UserFactory(email="dup@example.com")
|
||||
@pytest.mark.parametrize(
|
||||
"lookalike_email",
|
||||
[
|
||||
"dısk@example.com",
|
||||
"diſk@example.com",
|
||||
"disK@example.com",
|
||||
],
|
||||
ids=["dotless-i", "long-s", "kelvin-sign"],
|
||||
)
|
||||
def test_merge_does_not_merge_lookalike_emails(lookalike_email):
|
||||
"""A user whose email only matches the duplicates' email through Unicode case
|
||||
folding should not be merged with them, even when created last."""
|
||||
user1 = UserFactory(email="disk@example.com")
|
||||
user2 = UserFactory(email="disk@example.com")
|
||||
ra = UserResourceAccessFactory(user=user1)
|
||||
lookalike = UserFactory(email=lookalike_email)
|
||||
call_command("merge_duplicate_users")
|
||||
|
||||
assert not User.objects.filter(id=user1.id).exists()
|
||||
assert User.objects.filter(id=user2.id).exists()
|
||||
|
||||
user3 = UserFactory(email="joe@example.com")
|
||||
user4 = UserFactory(email="Joe@example.com")
|
||||
call_command("merge_duplicate_users")
|
||||
assert not User.objects.filter(id=user3.id).exists()
|
||||
assert User.objects.filter(id=user4.id).exists()
|
||||
|
||||
user4.refresh_from_db()
|
||||
assert user4.email.islower()
|
||||
ra.refresh_from_db()
|
||||
assert ra.user == user2
|
||||
lookalike.refresh_from_db()
|
||||
assert lookalike.email == lookalike_email
|
||||
assert not ResourceAccess.objects.filter(user=lookalike).exists()
|
||||
|
||||
|
||||
def test_merge_deletes_all_stale_users():
|
||||
@@ -409,10 +422,11 @@ def test_merge_does_not_merge_users_with_null_email():
|
||||
assert User.objects.filter(id=user2.id).exists()
|
||||
|
||||
|
||||
def test_merge_does_not_merge_users_with_blank_email():
|
||||
"""Users with empty-string email must never be merged together, even if multiple exist."""
|
||||
user1 = UserFactory(email="")
|
||||
user2 = UserFactory(email="")
|
||||
def test_merge_does_not_merge_users_with_legacy_blank_email():
|
||||
"""Users with a legacy empty-string email must never be merged together, even if
|
||||
multiple exist."""
|
||||
user1 = _user_with_stored_email("")
|
||||
user2 = _user_with_stored_email("")
|
||||
call_command("merge_duplicate_users")
|
||||
assert User.objects.filter(id=user1.id).exists()
|
||||
assert User.objects.filter(id=user2.id).exists()
|
||||
@@ -474,7 +488,7 @@ def test_merge_email_filter_no_match_does_nothing():
|
||||
|
||||
def test_merge_email_filter_is_case_insensitive():
|
||||
"""Command should match emails case-insensitively when filtering."""
|
||||
UserFactory(email="user1@Example.com")
|
||||
UserFactory(email="user1@Example.com")
|
||||
call_command("merge_duplicate_users", email_filter="@example.com")
|
||||
UserFactory(email="user1@example.com")
|
||||
UserFactory(email="user1@example.com")
|
||||
call_command("merge_duplicate_users", email_filter="@Example.COM")
|
||||
assert User.objects.filter(email="user1@example.com").count() == 1
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import json
|
||||
from unittest import mock
|
||||
|
||||
from django.core.exceptions import SuspiciousOperation
|
||||
from django.db import IntegrityError
|
||||
|
||||
import pytest
|
||||
@@ -38,6 +39,20 @@ def test_get_or_create_existing_user_is_not_audited(audit_events):
|
||||
assert not find_events(audit_events, "user.provision")
|
||||
|
||||
|
||||
def test_get_or_create_several_users_same_normalized_email(audit_events):
|
||||
"""Several users sharing the normalized email should be refused, not picked."""
|
||||
UserFactory(email="john.doe@example.com")
|
||||
UserFactory(email="John.Doe@example.com")
|
||||
|
||||
with pytest.raises(
|
||||
SuspiciousOperation, match="Multiple user accounts share a common email."
|
||||
):
|
||||
ProvisionalUserService().get_or_create(" JOHN.DOE@example.com", "my-client")
|
||||
|
||||
assert User.objects.count() == 2
|
||||
assert not find_events(audit_events, "user.provision")
|
||||
|
||||
|
||||
def test_get_or_create_creation_is_audited(audit_events):
|
||||
"""A created user is identified by its id, without sub nor email."""
|
||||
user, created = ProvisionalUserService().get_or_create(
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Tests for the user admin class."""
|
||||
|
||||
import pytest
|
||||
|
||||
from core import factories
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def static_storage_without_manifest(settings):
|
||||
"""Render the admin pages without collecting the static files first."""
|
||||
settings.STORAGES = {
|
||||
**settings.STORAGES,
|
||||
"staticfiles": {
|
||||
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _admin_client(client):
|
||||
"""Log a superuser in the client."""
|
||||
client.force_login(factories.UserFactory(is_staff=True, is_superuser=True))
|
||||
return client
|
||||
|
||||
|
||||
def test_admin_users_duplicate_email_filter(client):
|
||||
"""The filter should keep the users sharing their stored email, grouped by email
|
||||
and ordered by creation, but not the users whose email only matches through
|
||||
Unicode case folding."""
|
||||
first_jim = factories.UserFactory(email="jim@example.com")
|
||||
first_ann = factories.UserFactory(email="ann@example.com")
|
||||
second_jim = factories.UserFactory(email="JIM@example.com")
|
||||
second_ann = factories.UserFactory(email="ann@example.com")
|
||||
factories.UserFactory(email="jım@example.com")
|
||||
factories.UserFactory(email="bob@example.com")
|
||||
factories.UserFactory(email=None)
|
||||
factories.UserFactory(email=None)
|
||||
|
||||
response = _admin_client(client).get("/admin/core/user/?duplicate_email=yes")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [user.pk for user in response.context["cl"].result_list] == [
|
||||
first_ann.pk,
|
||||
second_ann.pk,
|
||||
first_jim.pk,
|
||||
second_jim.pk,
|
||||
]
|
||||
|
||||
|
||||
def test_admin_users_last_login(client):
|
||||
"""The last login of users should be shown in the list and on the change page."""
|
||||
user = factories.UserFactory()
|
||||
admin_client = _admin_client(client)
|
||||
|
||||
changelist = admin_client.get("/admin/core/user/")
|
||||
change = admin_client.get(f"/admin/core/user/{user.pk}/change/")
|
||||
|
||||
assert changelist.status_code == 200
|
||||
assert "column-last_login" in changelist.content.decode()
|
||||
assert change.status_code == 200
|
||||
assert "field-last_login" in change.content.decode()
|
||||
@@ -280,19 +280,22 @@ def test_menshen_unknown_user_not_created(settings):
|
||||
|
||||
|
||||
@responses.activate
|
||||
@pytest.mark.parametrize("fallback", [True, False])
|
||||
@pytest.mark.parametrize("existing_sub", [None, "other-sub"])
|
||||
def test_menshen_unknown_user_email_already_used(settings, existing_sub):
|
||||
def test_menshen_unknown_user_email_already_used(settings, existing_sub, fallback):
|
||||
"""An unknown sub should not create a second account with an email already
|
||||
used, by a provisional user or a user with another sub, when
|
||||
OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION=True."""
|
||||
used, by a provisional user or a user with another sub, whether
|
||||
OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION is set or not."""
|
||||
|
||||
settings.OIDC_CREATE_USER = True
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = True
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = fallback
|
||||
|
||||
existing_user = UserFactory(sub=existing_sub, email="User@Example.com")
|
||||
existing_user = UserFactory(sub=existing_sub, email="user@example.com")
|
||||
|
||||
responses.add(
|
||||
responses.POST, INTROSPECTION_ENDPOINT, json=_introspection("new-sub")
|
||||
responses.POST,
|
||||
INTROSPECTION_ENDPOINT,
|
||||
json=_introspection("new-sub", email=" User@Example.COM "),
|
||||
)
|
||||
|
||||
response = _create_room()
|
||||
@@ -305,6 +308,43 @@ def test_menshen_unknown_user_email_already_used(settings, existing_sub):
|
||||
assert not Room.objects.exists()
|
||||
|
||||
|
||||
@responses.activate
|
||||
@pytest.mark.parametrize("existing_sub", [None, "other-sub"])
|
||||
def test_menshen_unknown_user_email_already_used_allowed(
|
||||
audit_events, settings, existing_sub
|
||||
):
|
||||
"""An unknown sub should get its own account, even with an email already used,
|
||||
when OIDC_ALLOW_DUPLICATE_EMAILS=True."""
|
||||
|
||||
settings.OIDC_CREATE_USER = True
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = False
|
||||
settings.OIDC_ALLOW_DUPLICATE_EMAILS = True
|
||||
|
||||
existing_user = UserFactory(sub=existing_sub, email="user@example.com")
|
||||
|
||||
responses.add(
|
||||
responses.POST,
|
||||
INTROSPECTION_ENDPOINT,
|
||||
json=_introspection("new-sub", email=" User@Example.COM "),
|
||||
)
|
||||
|
||||
response = _create_room()
|
||||
|
||||
assert response.status_code == 201
|
||||
assert User.objects.count() == 2
|
||||
user = User.objects.get(sub="new-sub")
|
||||
assert user != existing_user
|
||||
assert user.email == "user@example.com"
|
||||
existing_user.refresh_from_db()
|
||||
assert existing_user.sub == existing_sub
|
||||
|
||||
[provision] = find_events(audit_events, "user.provision")
|
||||
|
||||
assert provision["event"]["outcome"] == "success"
|
||||
assert provision["lasuite"]["outcome"] == "success"
|
||||
assert provision["entity"]["target"]["id"] == str(user.pk)
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_menshen_is_audited_as_application(audit_events):
|
||||
"""A call with a Menshen token is made by the client Menshen reports, acting for
|
||||
@@ -382,10 +422,12 @@ def test_menshen_unknown_user_email_already_used_is_audited(audit_events, settin
|
||||
settings.OIDC_CREATE_USER = True
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = True
|
||||
|
||||
existing_user = UserFactory(sub=None, email="User@Example.com")
|
||||
existing_user = UserFactory(sub=None, email="user@example.com")
|
||||
|
||||
responses.add(
|
||||
responses.POST, INTROSPECTION_ENDPOINT, json=_introspection("new-sub")
|
||||
responses.POST,
|
||||
INTROSPECTION_ENDPOINT,
|
||||
json=_introspection("new-sub", email=" User@Example.COM "),
|
||||
)
|
||||
|
||||
response = _create_room()
|
||||
|
||||
@@ -184,7 +184,7 @@ def test_api_rooms_list_with_application_disabled(mock_rs_authenticate, settings
|
||||
|
||||
@responses.activate
|
||||
def test_api_rooms_list_with_invalid_rs_token(settings):
|
||||
"""Listing rooms with invalid resource server token should return 400."""
|
||||
"""Listing rooms with invalid resource server token should return 401."""
|
||||
|
||||
settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
|
||||
settings.OIDC_OP_URL = "https://oidc.example.com"
|
||||
@@ -202,9 +202,8 @@ def test_api_rooms_list_with_invalid_rs_token(settings):
|
||||
client.credentials(HTTP_AUTHORIZATION="Bearer invalid-token-123")
|
||||
response = client.get("/external-api/v1.0/rooms/")
|
||||
|
||||
# Return 400 instead of 401 because ResourceServerAuthentication raises
|
||||
# SuspiciousOperation when the introspected user is not active
|
||||
assert response.status_code == 400
|
||||
assert response.status_code == 401
|
||||
assert "introspected user is not active" in str(response.data).lower()
|
||||
|
||||
|
||||
def test_api_rooms_list_missing_scope():
|
||||
@@ -422,7 +421,7 @@ def test_api_rooms_retrieve_with_expired_token(settings):
|
||||
|
||||
@responses.activate
|
||||
def test_api_rooms_retrieve_with_invalid_rs_token(settings):
|
||||
"""Retrieving rooms with invalid resource server token should return 400."""
|
||||
"""Retrieving rooms with invalid resource server token should return 401."""
|
||||
|
||||
settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
|
||||
settings.OIDC_OP_URL = "https://oidc.example.com"
|
||||
@@ -443,9 +442,8 @@ def test_api_rooms_retrieve_with_invalid_rs_token(settings):
|
||||
client.credentials(HTTP_AUTHORIZATION="Bearer invalid-token-123")
|
||||
response = client.get(f"/external-api/v1.0/rooms/{room.id}/")
|
||||
|
||||
# Return 400 instead of 401 because ResourceServerAuthentication raises
|
||||
# SuspiciousOperation when the introspected user is not active
|
||||
assert response.status_code == 400
|
||||
assert response.status_code == 401
|
||||
assert "introspected user is not active" in str(response.data).lower()
|
||||
|
||||
|
||||
def test_api_rooms_retrieve_requires_scope():
|
||||
@@ -647,7 +645,7 @@ def test_api_rooms_create_with_expired_token(settings):
|
||||
|
||||
@responses.activate
|
||||
def test_api_rooms_create_with_invalid_rs_token(settings):
|
||||
"""Creating rooms with invalid resource server token should return 400."""
|
||||
"""Creating rooms with invalid resource server token should return 401."""
|
||||
|
||||
settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
|
||||
settings.OIDC_OP_URL = "https://oidc.example.com"
|
||||
@@ -665,9 +663,8 @@ def test_api_rooms_create_with_invalid_rs_token(settings):
|
||||
client.credentials(HTTP_AUTHORIZATION="Bearer invalid-token-123")
|
||||
response = client.post("/external-api/v1.0/rooms/")
|
||||
|
||||
# Return 400 instead of 401 because ResourceServerAuthentication raises
|
||||
# SuspiciousOperation when the introspected user is not active
|
||||
assert response.status_code == 400
|
||||
assert response.status_code == 401
|
||||
assert "introspected user is not active" in str(response.data).lower()
|
||||
|
||||
|
||||
def test_api_rooms_create_inactive_user():
|
||||
@@ -2971,6 +2968,36 @@ def test_api_rooms_missing_token_is_audited_as_denial(audit_events):
|
||||
assert event["url"]["path"] == "/external-api/v1.0/rooms/"
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_rooms_invalid_rs_token_is_audited_as_denial(audit_events, settings):
|
||||
"""A token refused by the identity provider is recorded as a denial."""
|
||||
settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
|
||||
settings.OIDC_OP_URL = "https://oidc.example.com"
|
||||
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://oidc.example.com/introspect",
|
||||
json={"iss": "https://oidc.example.com", "active": False},
|
||||
)
|
||||
|
||||
client = APIClient()
|
||||
client.credentials(HTTP_AUTHORIZATION="Bearer invalid-token-123")
|
||||
response = client.get("/external-api/v1.0/rooms/")
|
||||
|
||||
assert response.status_code == 401
|
||||
assert response.headers["WWW-Authenticate"] == "Bearer"
|
||||
responses.assert_call_count("https://oidc.example.com/introspect", 1)
|
||||
|
||||
[event] = find_events(audit_events, "room.list")
|
||||
|
||||
assert event["event"]["category"] == ["api", "authentication"]
|
||||
assert event["event"]["reason"] == "authentication_failed"
|
||||
assert event["lasuite"]["outcome"] == "denied"
|
||||
assert event["lasuite"]["actor"] == {"type": "anonymous"}
|
||||
assert event["http"]["response"] == {"status_code": 401}
|
||||
assert event["error"]["message"] == response.json()["detail"]
|
||||
|
||||
|
||||
def test_api_rooms_missing_scope_is_audited_as_denial(audit_events):
|
||||
"""A token without the required scope is a permission denial by the application."""
|
||||
user = UserFactory()
|
||||
|
||||
@@ -4,11 +4,12 @@ Unit tests for the User model
|
||||
|
||||
from unittest import mock
|
||||
|
||||
from django.contrib.auth.models import update_last_login
|
||||
from django.core.exceptions import ValidationError
|
||||
|
||||
import pytest
|
||||
|
||||
from core import factories
|
||||
from core import factories, models
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
@@ -117,12 +118,13 @@ def test_models_users_email_unique_when_sub_is_null():
|
||||
|
||||
|
||||
def test_models_users_email_unique_case_insensitive_when_sub_is_null():
|
||||
"""Email uniqueness should be case-insensitive among users with no sub (pending users)."""
|
||||
factories.UserFactory(sub=None, email="Test@example.com")
|
||||
"""Email uniqueness among users with no sub (pending users) should also ignore the
|
||||
case of non-ASCII letters, which the email normalization keeps."""
|
||||
factories.UserFactory(sub=None, email="test@éxample.com")
|
||||
with pytest.raises(
|
||||
ValidationError, match="Constraint “unique_email_when_sub_is_null” is violated."
|
||||
):
|
||||
factories.UserFactory(sub=None, email="test@example.com")
|
||||
factories.UserFactory(sub=None, email="test@Éxample.com")
|
||||
|
||||
|
||||
def test_models_users_email_not_unique_when_sub_is_set():
|
||||
@@ -175,3 +177,54 @@ def test_models_users_sub_blank_is_accepted():
|
||||
"""sub='' passes validation because blank=True; null is preferred but not enforced."""
|
||||
user = factories.UserFactory.build(sub="")
|
||||
user.full_clean()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"email, expected_email",
|
||||
[
|
||||
("John.Doe@Example.COM", "john.doe@example.com"),
|
||||
(" john.doe@example.com ", "john.doe@example.com"),
|
||||
],
|
||||
)
|
||||
def test_models_users_email_normalized_on_save(email, expected_email):
|
||||
"""The email of a user should be stripped and have its ASCII letters only lowercased
|
||||
when saved."""
|
||||
user = factories.UserFactory(email=email)
|
||||
user.refresh_from_db()
|
||||
assert user.email == expected_email
|
||||
|
||||
|
||||
def test_models_users_email_normalized_on_login():
|
||||
"""The last login update at login should store a legacy email normalized."""
|
||||
user = factories.UserFactory()
|
||||
models.User.objects.filter(pk=user.pk).update(email="John.Doe@Example.COM")
|
||||
user.refresh_from_db()
|
||||
|
||||
update_last_login(None, user)
|
||||
|
||||
user.refresh_from_db()
|
||||
assert user.last_login is not None
|
||||
assert user.email == "john.doe@example.com"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("email", ["", " "])
|
||||
def test_models_users_email_blank_stored_as_null(email):
|
||||
"""An empty or whitespace-only email should be stored as NULL."""
|
||||
user = factories.UserFactory(email=email)
|
||||
user.refresh_from_db()
|
||||
assert user.email is None
|
||||
|
||||
|
||||
def test_models_users_get_user_by_sub_or_email_active_preferred(settings):
|
||||
"""An active user should be preferred over an inactive one sharing its email."""
|
||||
settings.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION = True
|
||||
factories.UserFactory(email="john.doe@example.com", is_active=False)
|
||||
active_user = factories.UserFactory(email="John.Doe@example.com")
|
||||
factories.UserFactory(email="john.doe@example.com", is_active=False)
|
||||
|
||||
assert (
|
||||
models.User.objects.get_user_by_sub_or_email(
|
||||
"unknown-sub", "JOHN.DOE@example.com"
|
||||
)
|
||||
== active_user
|
||||
)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Unit tests for the OIDC_ALLOW_DUPLICATE_EMAILS setting."""
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from meet.settings import Test
|
||||
|
||||
|
||||
def test_oidc_allow_duplicate_emails_with_email_fallback_is_refused():
|
||||
"""Users identified by email cannot share one, so both settings stop the loading."""
|
||||
with (
|
||||
mock.patch.object(Test, "OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION", True),
|
||||
mock.patch.object(Test, "OIDC_ALLOW_DUPLICATE_EMAILS", True),
|
||||
pytest.raises(ValueError, match="OIDC_ALLOW_DUPLICATE_EMAILS"),
|
||||
):
|
||||
Test.post_setup()
|
||||
@@ -27,6 +27,7 @@ from core.utils import (
|
||||
NotificationError,
|
||||
create_livekit_client,
|
||||
generate_token,
|
||||
normalize_email,
|
||||
notify_participants,
|
||||
)
|
||||
|
||||
@@ -420,3 +421,14 @@ def test_notify_participants_success(mock_create_livekit_client):
|
||||
|
||||
# Verify aclose was called
|
||||
mock_api_instance.aclose.assert_called_once()
|
||||
|
||||
|
||||
def test_normalize_email():
|
||||
"""The "normalize_email" function should strip and lowercase ASCII letters only."""
|
||||
assert normalize_email(" John.Doe@Example.COM ") == "john.doe@example.com"
|
||||
assert normalize_email("Dısk@Example.com") == "dısk@example.com"
|
||||
assert normalize_email("Diſk@Example.com") == "diſk@example.com"
|
||||
assert normalize_email("DisK@Example.com") == "disK@example.com"
|
||||
assert normalize_email("Élise@Example.com") == "Élise@example.com"
|
||||
assert normalize_email("") == ""
|
||||
assert normalize_email(None) == ""
|
||||
|
||||
@@ -336,6 +336,26 @@ def generate_room_slug():
|
||||
return "-".join(parts)
|
||||
|
||||
|
||||
def is_uuid(value: str) -> bool:
|
||||
"""Tell whether a value parses as a UUID."""
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
_ASCII_LOWERCASE_TABLE = str.maketrans(string.ascii_uppercase, string.ascii_lowercase)
|
||||
|
||||
|
||||
def normalize_email(email):
|
||||
"""Strip an email and lowercase its ASCII letters only.
|
||||
|
||||
Emails are stored in this form and compared exactly.
|
||||
"""
|
||||
return (email or "").strip().translate(_ASCII_LOWERCASE_TABLE)
|
||||
|
||||
|
||||
def detect_mimetype(file_buffer: bytes, filename: str | None = None) -> str:
|
||||
"""
|
||||
Detect MIME type using multiple methods for better accuracy.
|
||||
|
||||
@@ -605,6 +605,14 @@ class Base(Configuration):
|
||||
environ_name="OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION",
|
||||
environ_prefix=None,
|
||||
)
|
||||
# WARNING: Enabling this setting allows multiple user accounts to share the same email
|
||||
# address. This may cause security issues and is not recommended for production use when
|
||||
# email is activated as fallback for identification
|
||||
OIDC_ALLOW_DUPLICATE_EMAILS = values.BooleanValue(
|
||||
default=False,
|
||||
environ_name="OIDC_ALLOW_DUPLICATE_EMAILS",
|
||||
environ_prefix=None,
|
||||
)
|
||||
OIDC_USER_SUB_FIELD_IMMUTABLE = values.BooleanValue(
|
||||
default=True, environ_name="OIDC_USER_SUB_FIELD_IMMUTABLE", environ_prefix=None
|
||||
)
|
||||
@@ -1623,6 +1631,15 @@ class Base(Configuration):
|
||||
if cls.LIVEKIT_API_TIMEOUT_SECONDS < 1:
|
||||
raise ValueError("LIVEKIT_API_TIMEOUT_SECONDS must be at least 1")
|
||||
|
||||
if (
|
||||
cls.OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION
|
||||
and cls.OIDC_ALLOW_DUPLICATE_EMAILS
|
||||
):
|
||||
raise ValueError(
|
||||
"Both OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION and "
|
||||
"OIDC_ALLOW_DUPLICATE_EMAILS cannot be set to True simultaneously."
|
||||
)
|
||||
|
||||
if (
|
||||
cls.SUMMARY_SERVICE_VERSION == 1
|
||||
and cls.SUMMARY_SERVICE_ENDPOINT is not None
|
||||
|
||||
@@ -7,7 +7,7 @@ build-backend = "uv_build"
|
||||
|
||||
[project]
|
||||
name = "meet"
|
||||
version = "1.36.3"
|
||||
version = "1.37.2"
|
||||
authors = [{ "name" = "DINUM", "email" = "dev@mail.numerique.gouv.fr" }]
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
@@ -33,7 +33,7 @@ dependencies = [
|
||||
"django-cors-headers==4.9.0",
|
||||
"django-countries==9.0.0",
|
||||
"django-filter==26.1",
|
||||
"django-lasuite[all]==0.0.29",
|
||||
"django-lasuite[all]==1.0.0",
|
||||
"django-parler==2.4",
|
||||
"redis==5.2.1",
|
||||
"django-redis==7.0.0",
|
||||
|
||||
Generated
+22
-11
@@ -837,26 +837,28 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "django-lasuite"
|
||||
version = "0.0.29"
|
||||
version = "1.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "django" },
|
||||
{ name = "djangorestframework" },
|
||||
{ name = "joserfc" },
|
||||
{ name = "mozilla-django-oidc" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "requests" },
|
||||
{ name = "requests-toolbelt" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f4/ab/72a79b7133fb2d17835bef40f45f7e6d2b4343469c37fd00386122e9068d/django_lasuite-0.0.29.tar.gz", hash = "sha256:831cb859e45e33963c9d89168c2de14f9844e805000b0f0d1512ffd0b9731a39", size = 35248, upload-time = "2026-08-25T13:03:43.597Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0f/cc/35335b826b039c43da68a0e624e154f60c3ac6c83f01f9b4ad6ce91faf03/django_lasuite-1.0.0.tar.gz", hash = "sha256:8921aaba5a78f1f9728cee2cc4596755478108f17146d4a2fccb393c043a05a3", size = 45815, upload-time = "2026-10-30T14:57:31.764Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/cf/86648643221816f18ebd5a3e59b6d2a85ee85b38b19c7abe939f798a2845/django_lasuite-0.0.29-py3-none-any.whl", hash = "sha256:33bcff90ced2709190ef01241badb17ceed08a32725b9cdd3105650b36da303b", size = 54899, upload-time = "2026-08-25T13:03:42.212Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/6f/d1df3f2881cda711e569b8e685ad3a846ca9c71226340597cee1c09beae0/django_lasuite-1.0.0-py3-none-any.whl", hash = "sha256:23d589be9496b1f484772b49f3c4c4a81e70affd537e3c50abf15303c84d7cb0", size = 68529, upload-time = "2026-10-30T14:57:30.416Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
all = [
|
||||
{ name = "celery" },
|
||||
{ name = "cryptography" },
|
||||
{ name = "django-configurations" },
|
||||
{ name = "djangorestframework" },
|
||||
{ name = "joserfc" },
|
||||
{ name = "mozilla-django-oidc" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "python-icap" },
|
||||
{ name = "requests" },
|
||||
{ name = "requests-toolbelt" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1358,7 +1360,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "meet"
|
||||
version = "1.36.3"
|
||||
version = "1.37.2"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
@@ -1440,7 +1442,7 @@ requires-dist = [
|
||||
{ name = "django-cors-headers", specifier = "==4.9.0" },
|
||||
{ name = "django-countries", specifier = "==9.0.0" },
|
||||
{ name = "django-filter", specifier = "==26.1" },
|
||||
{ name = "django-lasuite", extras = ["all"], specifier = "==0.0.29" },
|
||||
{ name = "django-lasuite", extras = ["all"], specifier = "==1.0.0" },
|
||||
{ name = "django-parler", specifier = "==2.4" },
|
||||
{ name = "django-pydantic-field", specifier = "==0.5.4" },
|
||||
{ name = "django-redis", specifier = "==7.0.0" },
|
||||
@@ -2258,6 +2260,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/a3/17c284b4f4d8ad50f0f9ba70ad8fcc35c777aeafcdbbffdd91bbdc5ab379/python_frontmatter-1.3.0-py3-none-any.whl", hash = "sha256:9f7dd9260bec99044219159a329f64f039087f9d1a2124c9442556f2fe6f82ec", size = 10562, upload-time = "2026-05-20T19:21:43.323Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-icap"
|
||||
version = "0.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5c/0a/123c30f4041eabb94c587d35fb64aa9498435d028a8577340a4e5adc5224/python_icap-0.2.0.tar.gz", hash = "sha256:627869ac29119074542de6ae95370914854fdb35ee0a13c8ae4c55847cfcb80e", size = 684516, upload-time = "2026-07-08T22:21:58.842Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/f2/b437a5e08b94dfc4cd8505cfc3e63922247e9ce3957707550ab08c514560/python_icap-0.2.0-py3-none-any.whl", hash = "sha256:b648d55b68abe387849a124d1b4a443801342a566afd2141f4515953ed4bd738", size = 70680, upload-time = "2026-07-08T22:21:57.307Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-magic"
|
||||
version = "0.4.27"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM node:22.23.4-alpine AS frontend-deps
|
||||
FROM --platform=$BUILDPLATFORM node:22.23.4-alpine AS frontend-deps
|
||||
|
||||
RUN apk upgrade --no-cache ca-certificates
|
||||
|
||||
@@ -53,8 +53,9 @@ USER root
|
||||
RUN apk upgrade --no-cache libexpat && \
|
||||
apk add --no-cache --upgrade 'pcre2>=10.49-r0' 'tiff>=4.7.2-r0' \
|
||||
'libx11>=1.8.14-r0' 'ca-certificates>=20261020-r0' \
|
||||
'brotli-libs>=1.2.1-r0' && \
|
||||
apk del curl
|
||||
'brotli-libs>=1.2.1-r0' 'musl-utils>=1.2.6-r3' && \
|
||||
apk del curl && \
|
||||
rm -f /usr/share/nginx/html/50x.html
|
||||
USER nginx
|
||||
|
||||
# Un-privileged user running the application
|
||||
|
||||
@@ -16,6 +16,17 @@ server {
|
||||
add_header Cache-Control "public, max-age=2592000";
|
||||
}
|
||||
|
||||
location = / {
|
||||
try_files /home.html /index.html;
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache";
|
||||
add_header Expires 0;
|
||||
}
|
||||
|
||||
location = /home.html {
|
||||
internal;
|
||||
}
|
||||
|
||||
# Serve static files
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
@@ -23,6 +34,7 @@ server {
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache"; # HTTP 1.0 header for backward compatibility
|
||||
add_header Expires 0;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
}
|
||||
|
||||
# Optionally, handle 404 errors by redirecting to index.html
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
|
||||
<link rel="manifest" href="/site.webmanifest">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="robots" content="noindex, nofollow" />
|
||||
<title>%VITE_APP_TITLE%</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
Generated
+2
-9
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "meet",
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "meet",
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"dependencies": {
|
||||
"@fontsource-variable/atkinson-hyperlegible-next": "5.3.0",
|
||||
"@fontsource-variable/lexend": "5.3.0",
|
||||
@@ -21,7 +21,6 @@
|
||||
"@remixicon/react": "4.9.0",
|
||||
"@tanstack/react-query": "5.101.4",
|
||||
"@timephy/rnnoise-wasm": "1.0.0",
|
||||
"crisp-sdk-web": "1.1.2",
|
||||
"hoofd": "1.7.3",
|
||||
"humanize-duration": "3.34.1",
|
||||
"i18next": "26.3.6",
|
||||
@@ -4563,12 +4562,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/crisp-sdk-web": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/crisp-sdk-web/-/crisp-sdk-web-1.1.2.tgz",
|
||||
"integrity": "sha512-qxSpiT4EGHnVEO6J/t1UURY6fg1ojgquLU0IIr7RNjZxXZs1R2e6AhAiFqxiGZ13Vb4o5VoUpATFU2TzX/ppOQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/cross-spawn": {
|
||||
"version": "7.0.6",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "meet",
|
||||
"private": true,
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "panda codegen && vite",
|
||||
@@ -29,7 +29,6 @@
|
||||
"@remixicon/react": "4.9.0",
|
||||
"@tanstack/react-query": "5.101.4",
|
||||
"@timephy/rnnoise-wasm": "1.0.0",
|
||||
"crisp-sdk-web": "1.1.2",
|
||||
"hoofd": "1.7.3",
|
||||
"humanize-duration": "3.34.1",
|
||||
"i18next": "26.3.6",
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
User-agent: GPTBot
|
||||
User-agent: OAI-SearchBot
|
||||
User-agent: ChatGPT-User
|
||||
User-agent: ClaudeBot
|
||||
User-agent: Claude-SearchBot
|
||||
User-agent: Claude-User
|
||||
User-agent: PerplexityBot
|
||||
User-agent: Perplexity-User
|
||||
User-agent: Google-Extended
|
||||
User-agent: Applebot-Extended
|
||||
User-agent: CCBot
|
||||
User-agent: Meta-ExternalAgent
|
||||
User-agent: Amazonbot
|
||||
User-agent: Bytespider
|
||||
Disallow: /
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
@@ -13,7 +13,9 @@ export interface ApiConfig {
|
||||
flags_api_host?: string
|
||||
}
|
||||
support?: {
|
||||
id: string
|
||||
widget_script_url?: string
|
||||
widget_api_url?: string
|
||||
widget_channel?: string
|
||||
help_article_transcript: string
|
||||
help_article_recording: string
|
||||
help_article_more_tools: string
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { useConfig } from '@/api/useConfig'
|
||||
import { useAnalytics } from '@/features/analytics/hooks/useAnalytics'
|
||||
import { useSupport } from '@/features/support/hooks/useSupport'
|
||||
import { useSyncUserPreferencesWithBackend } from '@/features/auth/api/useSyncUserPreferencesWithBackend'
|
||||
import { useEffect } from 'react'
|
||||
|
||||
@@ -8,10 +7,9 @@ export const AppInitialization = () => {
|
||||
const { data } = useConfig()
|
||||
useSyncUserPreferencesWithBackend()
|
||||
|
||||
const { analytics = {}, support = {}, custom_css_url = '' } = data ?? {}
|
||||
const { analytics = {}, custom_css_url = '' } = data ?? {}
|
||||
|
||||
useAnalytics(analytics)
|
||||
useSupport(support)
|
||||
|
||||
useEffect(() => {
|
||||
if (custom_css_url) {
|
||||
|
||||
@@ -32,6 +32,7 @@ export type LogCode =
|
||||
| 'fullscreen_failure'
|
||||
| 'publish_sources_failure'
|
||||
| 'disconnect_failure'
|
||||
| 'support_widget_failure'
|
||||
| 'generic_failure'
|
||||
|
||||
export const reportError = (
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { apiUrl } from '@/api/apiUrl'
|
||||
import { terminateSupportSession } from '@/features/support/hooks/useSupport'
|
||||
import { terminateAnalyticsSession } from '@/features/analytics/hooks/useAnalytics'
|
||||
|
||||
const logoutUrl = () => {
|
||||
@@ -8,6 +7,5 @@ const logoutUrl = () => {
|
||||
|
||||
export const logout = async () => {
|
||||
await terminateAnalyticsSession()
|
||||
terminateSupportSession()
|
||||
window.location.href = logoutUrl()
|
||||
}
|
||||
|
||||
+4
-9
@@ -2,23 +2,18 @@ import { RiQuestionLine } from '@remixicon/react'
|
||||
import { MenuItem } from 'react-aria-components'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { menuRecipe } from '@/primitives/menuRecipe'
|
||||
import {
|
||||
useIsSupportEnabled,
|
||||
openSupportChat,
|
||||
} from '@/features/support/hooks/useSupport'
|
||||
import { useSupportWidget } from '@/features/support/hooks/useSupportWidget'
|
||||
|
||||
export const SupportMenuItem = () => {
|
||||
const { t } = useTranslation('rooms', { keyPrefix: 'options.items' })
|
||||
const isSupportEnabled = useIsSupportEnabled()
|
||||
const { isEnabled, open } = useSupportWidget()
|
||||
|
||||
if (!isSupportEnabled) {
|
||||
return
|
||||
}
|
||||
if (!isEnabled) return
|
||||
|
||||
return (
|
||||
<MenuItem
|
||||
className={menuRecipe({ icon: true, variant: 'dark' }).item}
|
||||
onAction={openSupportChat}
|
||||
onAction={open}
|
||||
>
|
||||
<RiQuestionLine size={20} />
|
||||
{t('support')}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { RiQuestionLine } from '@remixicon/react'
|
||||
import { Button } from '@/primitives'
|
||||
import { Spinner } from '@/primitives/Spinner'
|
||||
import { useSupportWidget } from '../hooks/useSupportWidget'
|
||||
|
||||
export const SupportButton = () => {
|
||||
const { t } = useTranslation('global', { keyPrefix: 'support' })
|
||||
const { isEnabled, isPending, open } = useSupportWidget()
|
||||
|
||||
if (!isEnabled) return null
|
||||
|
||||
return (
|
||||
<Button
|
||||
square
|
||||
variant="secondaryText"
|
||||
aria-label={t('buttonLabel')}
|
||||
tooltip={t('buttonLabel')}
|
||||
isPending={isPending}
|
||||
onPress={open}
|
||||
>
|
||||
{isPending ? <Spinner size={24} /> : <RiQuestionLine />}
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import { useEffect } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useSnapshot } from 'valtio'
|
||||
import { useLocation } from 'wouter'
|
||||
import { RiCheckboxCircleFill, RiCloseLine } from '@remixicon/react'
|
||||
import { css } from '@/styled-system/css'
|
||||
import { Button, Text } from '@/primitives'
|
||||
import { announceToScreenReader } from '@/stores/screenReaderAnnouncer'
|
||||
import { supportStore } from '@/stores/support'
|
||||
import { subscribeToSupportWidget } from '../utils/supportWidget'
|
||||
import { ToastDuration } from '@/features/notifications/NotificationDuration'
|
||||
import { StyledToast } from '@/features/notifications/components/StyledToast'
|
||||
import { StyledToastContainer } from '@/features/notifications/components/StyledToastContainer'
|
||||
|
||||
const hide = () => {
|
||||
supportStore.showSentConfirmation = false
|
||||
}
|
||||
|
||||
export const SupportWidget = () => {
|
||||
const { t } = useTranslation('global', { keyPrefix: 'support.sent' })
|
||||
const { showSentConfirmation } = useSnapshot(supportStore)
|
||||
const [location] = useLocation()
|
||||
|
||||
useEffect(() => subscribeToSupportWidget(), [])
|
||||
|
||||
useEffect(hide, [location])
|
||||
|
||||
useEffect(() => {
|
||||
if (!showSentConfirmation) return
|
||||
announceToScreenReader(`${t('title')}. ${t('body')}`)
|
||||
const timeout = window.setTimeout(hide, ToastDuration.EXTRA_LONG)
|
||||
return () => window.clearTimeout(timeout)
|
||||
}, [showSentConfirmation, t])
|
||||
|
||||
if (!showSentConfirmation) return null
|
||||
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
position: 'fixed',
|
||||
top: '5rem',
|
||||
left: '50%',
|
||||
transform: 'translateX(-50%)',
|
||||
zIndex: 1000,
|
||||
})}
|
||||
>
|
||||
<StyledToastContainer>
|
||||
<StyledToast>
|
||||
<RiCheckboxCircleFill
|
||||
size={24}
|
||||
aria-hidden="true"
|
||||
className={css({ color: 'success.300', flexShrink: 0 })}
|
||||
/>
|
||||
<div>
|
||||
<Text margin={false} bold>
|
||||
{t('title')}
|
||||
</Text>
|
||||
<Text margin={false}>{t('body')}</Text>
|
||||
</div>
|
||||
<Button
|
||||
square
|
||||
size="sm"
|
||||
invisible
|
||||
aria-label={t('close')}
|
||||
onPress={hide}
|
||||
>
|
||||
<RiCloseLine color="white" />
|
||||
</Button>
|
||||
</StyledToast>
|
||||
</StyledToastContainer>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { type ApiUser } from '@/features/auth/api/ApiUser'
|
||||
import { useUser } from '@/features/auth/api/useUser'
|
||||
import { useConfig } from '@/api/useConfig'
|
||||
|
||||
type CrispSdk = (typeof import('crisp-sdk-web'))['Crisp']
|
||||
|
||||
let crisp: CrispSdk | undefined
|
||||
let crispPromise: Promise<CrispSdk> | undefined
|
||||
|
||||
const loadCrisp = (): Promise<CrispSdk> => {
|
||||
crispPromise ??= import('crisp-sdk-web')
|
||||
.then((module) => {
|
||||
crisp = module.Crisp
|
||||
return module.Crisp
|
||||
})
|
||||
.catch((error) => {
|
||||
crispPromise = undefined
|
||||
throw error
|
||||
})
|
||||
|
||||
return crispPromise
|
||||
}
|
||||
|
||||
export const openSupportChat = () => {
|
||||
if (!crisp?.isCrispInjected()) return
|
||||
crisp.chat.open()
|
||||
}
|
||||
|
||||
export const initializeSupportSession = (user: ApiUser) => {
|
||||
if (!crisp?.isCrispInjected()) return
|
||||
|
||||
const { id, email } = user
|
||||
crisp.setTokenId(`meet-${id}`)
|
||||
if (email) crisp.user.setEmail(email)
|
||||
}
|
||||
|
||||
export const terminateSupportSession = () => {
|
||||
if (!crisp?.isCrispInjected()) return
|
||||
|
||||
crisp.setTokenId()
|
||||
crisp.session.reset()
|
||||
}
|
||||
|
||||
export type useSupportProps = {
|
||||
id?: string
|
||||
isDisabled?: boolean
|
||||
}
|
||||
|
||||
const IDLE_TIMEOUT_MS = 10_000
|
||||
|
||||
const scheduleWhenIdle = (callback: () => void): (() => void) => {
|
||||
if (typeof window.requestIdleCallback === 'function') {
|
||||
const handle = window.requestIdleCallback(callback, {
|
||||
timeout: IDLE_TIMEOUT_MS,
|
||||
})
|
||||
return () => window.cancelIdleCallback(handle)
|
||||
}
|
||||
|
||||
const handle = window.setTimeout(callback, 1)
|
||||
return () => window.clearTimeout(handle)
|
||||
}
|
||||
|
||||
export const useSupport = ({ id, isDisabled }: useSupportProps) => {
|
||||
const { user } = useUser()
|
||||
const [isInjected, setIsInjected] = useState(
|
||||
() => crisp?.isCrispInjected() ?? false
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
if (!id || isDisabled) return
|
||||
|
||||
if (crisp?.isCrispInjected()) {
|
||||
setIsInjected(true)
|
||||
return
|
||||
}
|
||||
|
||||
let cancelled = false
|
||||
|
||||
const cancelIdle = scheduleWhenIdle(() => {
|
||||
void loadCrisp()
|
||||
.then((sdk) => {
|
||||
if (cancelled) return
|
||||
|
||||
if (!sdk.isCrispInjected()) {
|
||||
sdk.configure(id)
|
||||
sdk.setHideOnMobile(true)
|
||||
}
|
||||
|
||||
setIsInjected(true)
|
||||
})
|
||||
.catch((error) => {
|
||||
if (!cancelled) {
|
||||
console.error('Failed to initialize support chat', error)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
cancelIdle()
|
||||
}
|
||||
}, [id, isDisabled])
|
||||
|
||||
useEffect(() => {
|
||||
if (!user || !isInjected || isDisabled) return
|
||||
initializeSupportSession(user)
|
||||
}, [user, isInjected, isDisabled])
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
// Some users block the chat widget, so check its availability safely.
|
||||
const isCrispAvailable = () => {
|
||||
try {
|
||||
return !!window?.$crisp?.is
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export const useIsSupportEnabled = () => {
|
||||
const { data } = useConfig()
|
||||
return !!data?.support?.id && isCrispAvailable()
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useSnapshot } from 'valtio'
|
||||
import { useConfig } from '@/api/useConfig'
|
||||
import { useUser } from '@/features/auth/api/useUser'
|
||||
import { reportError } from '@/features/analytics/telemetry'
|
||||
import { announceToScreenReader } from '@/stores/screenReaderAnnouncer'
|
||||
import { supportStore } from '@/stores/support'
|
||||
import { openSupportWidget } from '../utils/supportWidget'
|
||||
|
||||
export const useSupportWidget = () => {
|
||||
const { t, i18n } = useTranslation('global', { keyPrefix: 'support' })
|
||||
const { data } = useConfig()
|
||||
const { user } = useUser()
|
||||
const { status } = useSnapshot(supportStore)
|
||||
|
||||
const {
|
||||
widget_script_url: scriptUrl,
|
||||
widget_api_url: api,
|
||||
widget_channel: channel,
|
||||
} = data?.support ?? {}
|
||||
const isEnabled = !!(scriptUrl && api && channel)
|
||||
|
||||
const open = () => {
|
||||
if (!isEnabled) return
|
||||
|
||||
openSupportWidget(scriptUrl, {
|
||||
api,
|
||||
channel,
|
||||
locale: i18n.language,
|
||||
email: user?.email,
|
||||
}).catch((error) => {
|
||||
reportError('support_widget_failure', error)
|
||||
announceToScreenReader(t('openError'), 'assertive')
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
isEnabled,
|
||||
isPending: status === 'pending',
|
||||
open,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { supportStore } from '@/stores/support'
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
_lasuite_widget?: unknown[]
|
||||
}
|
||||
}
|
||||
|
||||
type WidgetOptions = {
|
||||
api: string
|
||||
channel: string
|
||||
locale: string
|
||||
email?: string
|
||||
}
|
||||
|
||||
const ROOT_ID = 'lasuite-widget-support-root'
|
||||
const OPENED_EVENT = 'lasuite-widget-support-opened'
|
||||
const SUCCESS_EVENT = 'lasuite-widget-support-success'
|
||||
const CLOSED_EVENT = 'lasuite-widget-support-closed'
|
||||
const TIMEOUT_MS = 10_000
|
||||
|
||||
const WIDGET_CSS = `@media (min-width: 48em) {
|
||||
[role='dialog'] {
|
||||
top: 50%;
|
||||
right: auto;
|
||||
bottom: auto;
|
||||
left: 50%;
|
||||
transform: translate(-50%, -50%);
|
||||
}
|
||||
}
|
||||
[role='status'] {
|
||||
display: none;
|
||||
}`
|
||||
|
||||
const styleWidget = () => {
|
||||
const shadowRoot = document.getElementById(ROOT_ID)?.shadowRoot
|
||||
if (!shadowRoot) return
|
||||
const style = document.createElement('style')
|
||||
style.textContent = WIDGET_CSS
|
||||
shadowRoot.appendChild(style)
|
||||
}
|
||||
|
||||
let returnFocusTo: HTMLElement | null = null
|
||||
|
||||
// The widget removes the form without moving focus, which drops it on <body>.
|
||||
const restoreFocus = () => {
|
||||
const activeEl = document.activeElement
|
||||
const isFocusLost = activeEl === document.body || activeEl?.id === ROOT_ID
|
||||
if (isFocusLost && returnFocusTo?.isConnected) {
|
||||
returnFocusTo.focus({ preventScroll: true })
|
||||
}
|
||||
}
|
||||
|
||||
export const subscribeToSupportWidget = () => {
|
||||
const onOpened = () => {
|
||||
styleWidget()
|
||||
supportStore.status = 'open'
|
||||
}
|
||||
const onSuccess = () => {
|
||||
supportStore.showSentConfirmation = true
|
||||
restoreFocus()
|
||||
}
|
||||
const onClosed = () => {
|
||||
supportStore.status = 'closed'
|
||||
restoreFocus()
|
||||
returnFocusTo = null
|
||||
}
|
||||
|
||||
document.addEventListener(OPENED_EVENT, onOpened)
|
||||
document.addEventListener(SUCCESS_EVENT, onSuccess)
|
||||
document.addEventListener(CLOSED_EVENT, onClosed)
|
||||
return () => {
|
||||
document.removeEventListener(OPENED_EVENT, onOpened)
|
||||
document.removeEventListener(SUCCESS_EVENT, onSuccess)
|
||||
document.removeEventListener(CLOSED_EVENT, onClosed)
|
||||
}
|
||||
}
|
||||
|
||||
const resolveTrigger = () => {
|
||||
const activeEl = document.activeElement as HTMLElement | null
|
||||
if (activeEl?.getAttribute('role') === 'menuitem') {
|
||||
return document.querySelector<HTMLElement>('#room-options-trigger')
|
||||
}
|
||||
return activeEl
|
||||
}
|
||||
|
||||
let scriptPromise: Promise<void> | undefined
|
||||
|
||||
const loadScript = (src: string): Promise<void> => {
|
||||
scriptPromise ??= new Promise<void>((resolve, reject) => {
|
||||
const script = document.createElement('script')
|
||||
const timeout = window.setTimeout(
|
||||
() => fail(new Error(`Timed out loading ${src}`)),
|
||||
TIMEOUT_MS
|
||||
)
|
||||
const fail = (error: Error) => {
|
||||
window.clearTimeout(timeout)
|
||||
script.remove()
|
||||
scriptPromise = undefined
|
||||
reject(error)
|
||||
}
|
||||
|
||||
script.src = src
|
||||
script.async = true
|
||||
script.onload = () => {
|
||||
window.clearTimeout(timeout)
|
||||
resolve()
|
||||
}
|
||||
script.onerror = () => fail(new Error(`Failed to load ${src}`))
|
||||
document.body.appendChild(script)
|
||||
})
|
||||
|
||||
return scriptPromise
|
||||
}
|
||||
|
||||
const initWidget = (options: WidgetOptions): Promise<void> =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
const onOpened = () => {
|
||||
window.clearTimeout(timeout)
|
||||
resolve()
|
||||
}
|
||||
const timeout = window.setTimeout(() => {
|
||||
document.removeEventListener(OPENED_EVENT, onOpened)
|
||||
reject(new Error('Support widget did not open'))
|
||||
}, TIMEOUT_MS)
|
||||
|
||||
document.addEventListener(OPENED_EVENT, onOpened, { once: true })
|
||||
window._lasuite_widget?.push(['support', 'init', options])
|
||||
})
|
||||
|
||||
export const openSupportWidget = async (
|
||||
scriptUrl: string,
|
||||
options: WidgetOptions
|
||||
) => {
|
||||
if (supportStore.status !== 'closed') return
|
||||
|
||||
supportStore.status = 'pending'
|
||||
returnFocusTo = resolveTrigger()
|
||||
try {
|
||||
await loadScript(scriptUrl)
|
||||
await initWidget(options)
|
||||
} catch (error) {
|
||||
supportStore.status = 'closed'
|
||||
returnFocusTo = null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { HStack, Stack } from '@/styled-system/jsx'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Button, Text } from '@/primitives'
|
||||
import { SettingsButton } from '@/features/settings'
|
||||
import { SupportButton } from '@/features/support/components/SupportButton'
|
||||
import { useUser } from '@/features/auth/api/useUser'
|
||||
import { useMatchesRoute } from '@/navigation/useMatchesRoute'
|
||||
import { FeedbackBanner } from '@/components/FeedbackBanner'
|
||||
@@ -153,6 +154,7 @@ export const Header = () => {
|
||||
</header>
|
||||
<nav>
|
||||
<Stack gap={1} direction="row" align="center">
|
||||
<SupportButton />
|
||||
{isLoggedIn === false &&
|
||||
!isHome &&
|
||||
!isLegalTerms &&
|
||||
|
||||
@@ -6,6 +6,7 @@ import { useSnapshot } from 'valtio'
|
||||
import { Footer } from '@/layout/Footer'
|
||||
import { ScreenReaderAnnouncer } from '@/primitives'
|
||||
import { SkipLink, MAIN_CONTENT_ID } from './SkipLink'
|
||||
import { SupportWidget } from '@/features/support/components/SupportWidget'
|
||||
|
||||
export type Layout = 'fullpage' | 'centered'
|
||||
|
||||
@@ -46,6 +47,7 @@ export const Layout = ({ children }: { children: ReactNode }) => {
|
||||
})}
|
||||
>
|
||||
<ScreenReaderAnnouncer />
|
||||
<SupportWidget />
|
||||
{children}
|
||||
</main>
|
||||
</div>
|
||||
|
||||
@@ -27,6 +27,15 @@
|
||||
},
|
||||
"selected": "ausgewählt",
|
||||
"submit": "OK",
|
||||
"support": {
|
||||
"buttonLabel": "Support kontaktieren",
|
||||
"openError": "Das Kontaktformular konnte nicht geöffnet werden. Bitte versuchen Sie es später erneut.",
|
||||
"sent": {
|
||||
"title": "Nachricht gesendet",
|
||||
"body": "Das Support-Team hat Ihre Nachricht erhalten.",
|
||||
"close": "Schließen"
|
||||
}
|
||||
},
|
||||
"footer": {
|
||||
"links": {
|
||||
"legifrance": "legifrance.gouv.fr",
|
||||
|
||||
@@ -27,6 +27,15 @@
|
||||
},
|
||||
"selected": "selected",
|
||||
"submit": "OK",
|
||||
"support": {
|
||||
"buttonLabel": "Contact support",
|
||||
"openError": "The contact form could not be opened. Please try again later.",
|
||||
"sent": {
|
||||
"title": "Message sent",
|
||||
"body": "The support team has received your message.",
|
||||
"close": "Close"
|
||||
}
|
||||
},
|
||||
"footer": {
|
||||
"links": {
|
||||
"legifrance": "legifrance.gouv.fr",
|
||||
|
||||
@@ -27,6 +27,15 @@
|
||||
},
|
||||
"selected": "seleccionado",
|
||||
"submit": "Aceptar",
|
||||
"support": {
|
||||
"buttonLabel": "Contactar con el soporte",
|
||||
"openError": "No se ha podido abrir el formulario de contacto. Inténtelo de nuevo más tarde.",
|
||||
"sent": {
|
||||
"title": "Mensaje enviado",
|
||||
"body": "El equipo de soporte ha recibido su mensaje.",
|
||||
"close": "Cerrar"
|
||||
}
|
||||
},
|
||||
"footer": {
|
||||
"links": {
|
||||
"legifrance": "legifrance.gouv.fr",
|
||||
|
||||
@@ -27,6 +27,15 @@
|
||||
},
|
||||
"selected": "sélectionné",
|
||||
"submit": "OK",
|
||||
"support": {
|
||||
"buttonLabel": "Contacter l'aide",
|
||||
"openError": "Impossible d'ouvrir le formulaire de contact. Réessayez plus tard.",
|
||||
"sent": {
|
||||
"title": "Message envoyé",
|
||||
"body": "L'équipe support a bien reçu votre message.",
|
||||
"close": "Fermer"
|
||||
}
|
||||
},
|
||||
"footer": {
|
||||
"links": {
|
||||
"legifrance": "legifrance.gouv.fr",
|
||||
|
||||
@@ -26,6 +26,15 @@
|
||||
},
|
||||
"selected": "geselecteerd",
|
||||
"submit": "OK",
|
||||
"support": {
|
||||
"buttonLabel": "Contact ondersteuning",
|
||||
"openError": "Het contactformulier kon niet worden geopend. Probeer het later opnieuw.",
|
||||
"sent": {
|
||||
"title": "Bericht verzonden",
|
||||
"body": "Het supportteam heeft je bericht ontvangen.",
|
||||
"close": "Sluiten"
|
||||
}
|
||||
},
|
||||
"footer": {
|
||||
"links": {
|
||||
"legifrance": "legifrance.gouv.fr",
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { proxy } from 'valtio'
|
||||
|
||||
export type SupportWidgetStatus = 'closed' | 'pending' | 'open'
|
||||
|
||||
type State = {
|
||||
status: SupportWidgetStatus
|
||||
showSentConfirmation: boolean
|
||||
}
|
||||
|
||||
export const supportStore = proxy<State>({
|
||||
status: 'closed',
|
||||
showSentConfirmation: false,
|
||||
})
|
||||
@@ -72,14 +72,6 @@ main#main-content :is(h1, h2, h3)[tabindex='-1']:focus {
|
||||
}
|
||||
}
|
||||
|
||||
body:has(.lk-video-conference) #crisp-chatbox-button {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
body:has(.lk-video-conference) #crisp-chatbox > * > div[role='button'] {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
html:has(.lk-video-conference) {
|
||||
scrollbar-gutter: auto !important;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { defineConfig, loadEnv } from 'vite'
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { defineConfig, loadEnv, type Plugin } from 'vite'
|
||||
import react from '@vitejs/plugin-react'
|
||||
import { visualizer } from 'rollup-plugin-visualizer'
|
||||
import svgr from 'vite-plugin-svgr'
|
||||
@@ -25,6 +26,23 @@ if (mediapipeVersion !== livekitMediapipeVersion) {
|
||||
)
|
||||
}
|
||||
|
||||
const NOINDEX_TAG = '<meta name="robots" content="noindex, nofollow" />'
|
||||
|
||||
const homePage = (): Plugin => ({
|
||||
name: 'home-page',
|
||||
apply: 'build',
|
||||
writeBundle({ dir = 'dist' }) {
|
||||
const html = readFileSync(join(dir, 'index.html'), 'utf-8')
|
||||
if (!html.includes(NOINDEX_TAG)) {
|
||||
throw new Error(
|
||||
`${dir}/index.html does not contain ${NOINDEX_TAG}, so home.html ` +
|
||||
`cannot be generated from it. The two must stay in sync.`
|
||||
)
|
||||
}
|
||||
writeFileSync(join(dir, 'home.html'), html.replace(NOINDEX_TAG, ''))
|
||||
},
|
||||
})
|
||||
|
||||
// https://vitejs.dev/config/
|
||||
export default defineConfig(({ mode }) => {
|
||||
const env = loadEnv(mode, process.cwd())
|
||||
@@ -66,6 +84,7 @@ export default defineConfig(({ mode }) => {
|
||||
},
|
||||
],
|
||||
}),
|
||||
homePage(),
|
||||
env.VITE_ANALYZE === 'true' &&
|
||||
visualizer({
|
||||
open: true,
|
||||
|
||||
@@ -163,7 +163,7 @@ backend:
|
||||
LIVEKIT_DEFAULT_VIDEO_CODEC: vp9
|
||||
FRONTEND_IDLE_DISCONNECT_WARNING_DELAY: 9000
|
||||
FRONTEND_SILENCE_LIVEKIT_DEBUG: False
|
||||
FRONTEND_SUPPORT: "{'id': '58ea6697-8eba-4492-bc59-ad6562585041', 'help_article_transcript': 'https://lasuite.crisp.help/fr/article/visio-transcript-1sjq43x', 'help_article_recording': 'https://lasuite.crisp.help/fr/article/visio-enregistrement-wgc8o0', 'help_article_more_tools': 'https://lasuite.crisp.help/fr/article/visio-tools-bvxj23'}"
|
||||
FRONTEND_SUPPORT: "{'help_article_transcript': 'https://docs.numerique.gouv.fr/docs/ba213749-2379-421c-b2ea-e4bd0840ea9b', 'help_article_recording': 'https://docs.numerique.gouv.fr/docs/a4e12e41-2974-472c-84a9-c00d12c10769', 'help_article_more_tools': 'https://docs.numerique.gouv.fr/docs/75fa98c7-5af1-433e-a611-e38143db26d0'}"
|
||||
FRONTEND_FEEDBACK: "{'url': 'https://grist.numerique.gouv.fr/o/docs/cbMv4G7pLY3Z/USER-RESEARCH-or-LA-SUITE/f/26'}"
|
||||
FRONTEND_DOCUMENTATION_URL: "https://docs.numerique.gouv.fr/docs/7c5bd65d-3c21-486f-bce1-26e0a921d642/"
|
||||
FRONTEND_TECHNICAL_DOCUMENTATION_URL: "https://docs.numerique.gouv.fr/docs/f2baa1b9-f29e-4d58-959d-65d4376fc6b8/"
|
||||
|
||||
@@ -51,6 +51,7 @@ data:
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache" always;
|
||||
add_header Expires 0 always;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
set $ms_domains "https://*.live.com https://*.office.com https://*.microsoft.com https://*.office365.com https://*.sharepoint.com";
|
||||
|
||||
@@ -84,6 +85,17 @@ data:
|
||||
add_header Cache-Control "public, max-age=2592000";
|
||||
}
|
||||
|
||||
location = / {
|
||||
try_files /home.html /index.html;
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache";
|
||||
add_header Expires 0;
|
||||
}
|
||||
|
||||
location = /home.html {
|
||||
internal;
|
||||
}
|
||||
|
||||
# Serve static files
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
@@ -91,6 +103,7 @@ data:
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache"; # HTTP 1.0 header for backward compatibility
|
||||
add_header Expires 0;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
}
|
||||
|
||||
# Optionally, handle 404 errors by redirecting to index.html
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "mail_mjml",
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "mail_mjml",
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@html-to/text-cli": "0.6.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mail_mjml",
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"description": "An util to generate html and text django's templates from mjml templates",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "sdk",
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "sdk",
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"license": "ISC",
|
||||
"workspaces": [
|
||||
"./library",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "sdk",
|
||||
"version": "1.36.3",
|
||||
"version": "1.37.2",
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
FROM python:3.13.16-alpine3.24 AS base
|
||||
|
||||
RUN apk add --no-cache --upgrade 'ca-certificates>=20261020-r0'
|
||||
RUN apk add --no-cache --upgrade 'ca-certificates>=20261020-r0' \
|
||||
'musl-utils>=1.2.6-r3'
|
||||
|
||||
|
||||
# Install ffmpeg for audio/video processing (format conversion, extraction, compression)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
[project]
|
||||
name = "summary"
|
||||
version = "1.36.3"
|
||||
version = "1.37.2"
|
||||
requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"certifi>=2026.10.19",
|
||||
|
||||
@@ -5,7 +5,6 @@ from datetime import datetime, timezone
|
||||
|
||||
from celery.result import AsyncResult
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import ValidationError
|
||||
|
||||
from summary.core.analytics import get_analytics
|
||||
from summary.core.celery_worker import (
|
||||
@@ -14,9 +13,7 @@ from summary.core.celery_worker import (
|
||||
summarize_v2_task,
|
||||
)
|
||||
from summary.core.config import AuthorizedTenant, get_settings
|
||||
from summary.core.exceptions import CleanedValidationError
|
||||
from summary.core.models import SummarizeTaskApiRequest, TranscribeTaskApiRequest
|
||||
from summary.core.privacy import dump_for_transport
|
||||
from summary.core.security import verify_tenant_api_key_v2
|
||||
from summary.core.shared_models import (
|
||||
SummarizeWebhookFailurePayload,
|
||||
@@ -55,7 +52,7 @@ async def create_transcribe_task_v2(
|
||||
task = process_audio_transcribe_v2_task.apply_async(
|
||||
args=[
|
||||
{
|
||||
**dump_for_transport(request),
|
||||
**request.model_dump(),
|
||||
"tenant_id": request_tenant.id,
|
||||
"received_at": datetime.now(timezone.utc),
|
||||
}
|
||||
@@ -128,11 +125,7 @@ async def get_transcribe_job_status(
|
||||
|
||||
if task.status == "SUCCESS":
|
||||
result = task.result
|
||||
try:
|
||||
payload = TranscribeWebhookSuccessPayload.model_validate(result)
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
return dump_for_transport(payload)
|
||||
return TranscribeWebhookSuccessPayload.model_validate(result).model_dump()
|
||||
|
||||
if task.status == "FAILURE":
|
||||
return TranscribeWebhookFailurePayload(
|
||||
@@ -162,11 +155,7 @@ async def get_summarize_job_status(
|
||||
|
||||
if task.status == "SUCCESS":
|
||||
result = task.result
|
||||
try:
|
||||
payload = SummarizeWebhookSuccessPayload.model_validate(result)
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
return dump_for_transport(payload)
|
||||
return SummarizeWebhookSuccessPayload.model_validate(result).model_dump()
|
||||
|
||||
if task.status == "FAILURE":
|
||||
return SummarizeWebhookFailurePayload(
|
||||
|
||||
@@ -5,6 +5,7 @@ import socket
|
||||
import time
|
||||
from collections import Counter
|
||||
from functools import cached_property, lru_cache
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import redis
|
||||
from celery.utils.log import get_task_logger
|
||||
@@ -12,7 +13,6 @@ from posthog import Posthog
|
||||
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.models import SummarizeTaskJob, TranscribeTaskJob
|
||||
from summary.core.privacy import redact_url
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
settings = get_settings()
|
||||
@@ -54,8 +54,8 @@ class Analytics:
|
||||
self._client.capture(
|
||||
event_name, distinct_id=distinct_id, properties=properties
|
||||
)
|
||||
except Exception:
|
||||
raise AnalyticsException("Failed to capture analytics event") from None
|
||||
except Exception as e:
|
||||
raise AnalyticsException("Failed to capture analytics event") from e
|
||||
|
||||
def is_feature_enabled(self, feature_name: str, distinct_id: str = None) -> bool:
|
||||
"""Check if a feature flag is enabled for a user."""
|
||||
@@ -64,8 +64,8 @@ class Analytics:
|
||||
|
||||
try:
|
||||
return self._client.feature_enabled(feature_name, distinct_id)
|
||||
except Exception:
|
||||
logger.error("Error checking feature flag %s", feature_name)
|
||||
except Exception as e:
|
||||
logger.error("Error checking feature flag %s: %s", feature_name, e)
|
||||
return False
|
||||
|
||||
|
||||
@@ -133,7 +133,8 @@ class MetadataManager:
|
||||
}
|
||||
|
||||
if isinstance(task_payload, TranscribeTaskJob):
|
||||
clean_url = redact_url(task_payload.cloud_storage_url.get_secret_value())
|
||||
parts = urlsplit(task_payload.cloud_storage_url)
|
||||
clean_url = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||
initial_metadata["source_url"] = clean_url
|
||||
initial_metadata["asr_model"] = settings.whisperx_asr_model
|
||||
elif isinstance(task_payload, SummarizeTaskJob):
|
||||
|
||||
@@ -3,23 +3,20 @@
|
||||
# ruff: noqa: PLR0913
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import requests
|
||||
from celery import Celery, Task, signals
|
||||
from celery import Celery, signals
|
||||
from celery.utils.log import get_task_logger
|
||||
from openai.types.audio import Transcription
|
||||
from pydantic import SecretStr, ValidationError
|
||||
from requests import exceptions
|
||||
|
||||
from summary.core.analytics import MetadataManager, get_analytics
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.docs_service import create_document_in_lasuite_docs
|
||||
from summary.core.exceptions import CleanedValidationError
|
||||
from summary.core.file_service import (
|
||||
CorruptedAudioFile,
|
||||
FileService,
|
||||
@@ -34,7 +31,6 @@ from summary.core.models import (
|
||||
SummarizeTaskJob,
|
||||
TranscribeTaskJob,
|
||||
)
|
||||
from summary.core.privacy import dump_for_transport, redact_url
|
||||
from summary.core.prompt import (
|
||||
FORMAT_NEXT_STEPS,
|
||||
FORMAT_PLAN,
|
||||
@@ -67,24 +63,8 @@ metadata_manager = MetadataManager()
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def _reveal(secret: SecretStr | None) -> str | None:
|
||||
"""Return the underlying value of an optional secret."""
|
||||
return secret.get_secret_value() if secret is not None else None
|
||||
|
||||
|
||||
class PrivateTask(Task):
|
||||
"""Hide task arguments in Celery logs and monitoring events."""
|
||||
|
||||
def apply_async(self, args=None, kwargs=None, **options):
|
||||
"""Publish with redacted display representations."""
|
||||
options.update(argsrepr="[Filtered]", kwargsrepr="[Filtered]")
|
||||
return super().apply_async(args=args, kwargs=kwargs, **options)
|
||||
|
||||
|
||||
celery = Celery(
|
||||
__name__,
|
||||
task_cls=PrivateTask,
|
||||
broker=settings.celery_broker_url,
|
||||
backend=settings.celery_result_backend,
|
||||
broker_connection_retry_on_startup=True,
|
||||
@@ -96,28 +76,10 @@ celery = Celery(
|
||||
celery.config_from_object("summary.core.celery_config")
|
||||
|
||||
|
||||
class CeleryLogPrivacyFilter(logging.Filter):
|
||||
"""Prevent Celery trace logs from serializing task results or exceptions."""
|
||||
|
||||
def filter(self, record):
|
||||
"""Keep task identity and state without upstream values."""
|
||||
if record.levelno < logging.INFO:
|
||||
return True
|
||||
data = getattr(record, "data", None)
|
||||
if isinstance(data, dict):
|
||||
record.msg = "Task %s[%s] finished or retried (details omitted)"
|
||||
record.args = (data.get("name", "unknown"), data.get("id", "unknown"))
|
||||
record.data = {}
|
||||
record.exc_info = None
|
||||
record.exc_text = None
|
||||
return True
|
||||
|
||||
|
||||
@signals.celeryd_init.connect
|
||||
def init_celery_sentry(**_kwargs):
|
||||
"""Initialize Sentry in the Celery worker."""
|
||||
init_sentry()
|
||||
logging.getLogger("celery.app.trace").addFilter(CeleryLogPrivacyFilter())
|
||||
|
||||
|
||||
file_service = FileService()
|
||||
@@ -200,9 +162,9 @@ def transcribe_audio(
|
||||
try:
|
||||
res.raise_for_status()
|
||||
except requests.exceptions.HTTPError:
|
||||
logger.error("WhisperX transcription failed")
|
||||
logger.exception("WhisperX transcription failed")
|
||||
# We reraise the error so that it can be retried by celery
|
||||
raise requests.exceptions.HTTPError("WhisperX request failed") from None
|
||||
raise
|
||||
|
||||
transcription_json: dict[str, Any] = res.json()
|
||||
# We remove the "usage" key from the transcription_json dictionary
|
||||
@@ -213,16 +175,13 @@ def transcribe_audio(
|
||||
# We force the use of the Transcription model here
|
||||
# to avoid changing too much code for now.
|
||||
# Note that it should be WhisperXResponse instead.
|
||||
try:
|
||||
transcription = Transcription.model_validate(
|
||||
# We add a dummy "text" to make the model validate,
|
||||
# Some API responses lack the "text" key.
|
||||
{"text": "", **transcription_json},
|
||||
extra="allow",
|
||||
strict=False,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
transcription = Transcription.model_validate(
|
||||
# We add a dummy "text" to make the model validate,
|
||||
# Some API responses lack the "text" key.
|
||||
{"text": "", **transcription_json},
|
||||
extra="allow",
|
||||
strict=False,
|
||||
)
|
||||
|
||||
# Logging
|
||||
transcription_duration = round(time.time() - transcription_start_time, 2)
|
||||
@@ -240,9 +199,9 @@ def transcribe_audio(
|
||||
if raises:
|
||||
raise e
|
||||
redacted_cloud_storage_url = (
|
||||
redact_url(cloud_storage_url) if cloud_storage_url else None
|
||||
cloud_storage_url.split("?", 1)[0] if cloud_storage_url else None
|
||||
)
|
||||
logger.error(
|
||||
logger.exception(
|
||||
("Unexpected error while preparing file %s "),
|
||||
redacted_cloud_storage_url,
|
||||
)
|
||||
@@ -271,7 +230,7 @@ def resolve_speaker_identities_and_apply_to(
|
||||
logger.debug("Running resolve_speaker_identities")
|
||||
try:
|
||||
metadata = file_service.read_cloud_storage_json(
|
||||
recording_metadata.cloud_storage_url.get_secret_value()
|
||||
recording_metadata.cloud_storage_url
|
||||
)
|
||||
speaker_mapping = resolve_speaker_identities(
|
||||
metadata,
|
||||
@@ -282,18 +241,21 @@ def resolve_speaker_identities_and_apply_to(
|
||||
new_transcription = speaker_mapping.apply_to(transcription.model_dump())
|
||||
return WhisperXResponse.model_validate(new_transcription)
|
||||
|
||||
except FileServiceException:
|
||||
except FileServiceException as exc:
|
||||
logger.error(
|
||||
"Error reading metadata for task %s; skipping speaker assignment.",
|
||||
"Error reading metadata for task %s; skipping speaker assignment."
|
||||
" Error: %s",
|
||||
task_id,
|
||||
exc,
|
||||
)
|
||||
return transcription
|
||||
|
||||
except Exception:
|
||||
logger.error(
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"resolve_speaker_identities failed for task %s; skipping"
|
||||
" speaker assignment.",
|
||||
" speaker assignment. Error: %s",
|
||||
task_id,
|
||||
exc,
|
||||
)
|
||||
return transcription
|
||||
|
||||
@@ -340,17 +302,6 @@ def format_actions(llm_output: dict) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def parse_llm_json(value: str) -> dict:
|
||||
"""Parse an LLM response without exposing it in decoding errors."""
|
||||
try:
|
||||
result = json.loads(value)
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError
|
||||
return result
|
||||
except (TypeError, ValueError):
|
||||
raise LLMException("Invalid LLM response") from None
|
||||
|
||||
|
||||
def summarize_transcription_internals(
|
||||
*, distinct_id: str, transcript: str, session_id: str
|
||||
) -> str:
|
||||
@@ -391,13 +342,13 @@ def summarize_transcription_internals(
|
||||
)
|
||||
logger.info("Plan generated")
|
||||
|
||||
res = parse_llm_json(parts)
|
||||
res = json.loads(parts)
|
||||
parts = res.get("titles", [])
|
||||
logger.info("Parts to summarize: %s", len(parts))
|
||||
logger.info("Parts to summarize: %s", parts)
|
||||
parts_summarized = []
|
||||
for i, part in enumerate(parts):
|
||||
for part in parts:
|
||||
prompt_user_part = PROMPT_USER_PART.format(part=part, transcript=transcript)
|
||||
logger.info("Summarizing part %s", i + 1)
|
||||
logger.info("Summarizing part: %s", part)
|
||||
parts_summarized.append(
|
||||
llm_service.call(PROMPT_SYSTEM_PART, prompt_user_part, name="part")
|
||||
)
|
||||
@@ -413,7 +364,7 @@ def summarize_transcription_internals(
|
||||
response_format=FORMAT_NEXT_STEPS,
|
||||
)
|
||||
|
||||
next_steps = format_actions(parse_llm_json(next_steps))
|
||||
next_steps = format_actions(json.loads(next_steps))
|
||||
|
||||
logger.info("Next steps generated")
|
||||
|
||||
@@ -486,11 +437,9 @@ def call_webhook_v2_task(
|
||||
tenant_id: str,
|
||||
):
|
||||
"""Calls a webhook asynchrously (retry handled by celery)."""
|
||||
try:
|
||||
validated_payload = webhook_payload_adapter.validate_python(payload)
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
call_webhook_v2(payload=validated_payload, tenant_id=tenant_id)
|
||||
call_webhook_v2(
|
||||
payload=webhook_payload_adapter.validate_python(payload), tenant_id=tenant_id
|
||||
)
|
||||
|
||||
|
||||
@celery.task(
|
||||
@@ -516,10 +465,7 @@ def process_audio_transcribe_v2_task(
|
||||
self: Celery task instance (passed on with bind=True)
|
||||
payload: Serialized dictionary of TranscribeSummarizeTaskCreationV2
|
||||
"""
|
||||
try:
|
||||
payload = TranscribeTaskJob.model_validate(payload)
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
payload = TranscribeTaskJob.model_validate(payload)
|
||||
logger.info(
|
||||
"Transcribing for object received | Owner: %s",
|
||||
payload.user_sub,
|
||||
@@ -531,7 +477,7 @@ def process_audio_transcribe_v2_task(
|
||||
transcription_res = WhisperXResponse(
|
||||
**transcribe_audio( # type: ignore
|
||||
task_id=job_id,
|
||||
cloud_storage_url=payload.cloud_storage_url.get_secret_value(),
|
||||
cloud_storage_url=payload.cloud_storage_url,
|
||||
language=payload.language,
|
||||
raises=True,
|
||||
).model_dump()
|
||||
@@ -568,8 +514,8 @@ def process_audio_transcribe_v2_task(
|
||||
transcription_res.model_dump(),
|
||||
payload.context_language,
|
||||
payload.language,
|
||||
_reveal(payload.push_to_docs_config.download_link),
|
||||
_reveal(payload.push_to_docs_config.form_link),
|
||||
payload.push_to_docs_config.download_link,
|
||||
payload.push_to_docs_config.form_link,
|
||||
)
|
||||
|
||||
create_document_in_lasuite_docs(
|
||||
@@ -610,22 +556,18 @@ def process_audio_transcribe_v2_task(
|
||||
transcription_data_url=file_service.get_transcript_signed_url(job_id),
|
||||
)
|
||||
call_webhook_v2_task.apply_async(
|
||||
args=[dump_for_transport(success_payload), payload.tenant_id]
|
||||
args=[success_payload.model_dump(), payload.tenant_id]
|
||||
)
|
||||
metadata_manager.capture(job_id, settings.posthog_transcript_success)
|
||||
|
||||
return dump_for_transport(success_payload)
|
||||
return success_payload.model_dump()
|
||||
|
||||
|
||||
@signals.task_prerun.connect(sender=process_audio_transcribe_v2_task)
|
||||
def task_started_transcript(task_id=None, task=None, args=None, **kwargs):
|
||||
"""Signal handler called before task execution begins."""
|
||||
if args:
|
||||
try:
|
||||
payload = TranscribeTaskJob.model_validate(args[0])
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
metadata_manager.create(task_id, payload)
|
||||
metadata_manager.create(task_id, TranscribeTaskJob.model_validate(args[0]))
|
||||
|
||||
|
||||
@signals.task_retry.connect(sender=process_audio_transcribe_v2_task)
|
||||
@@ -685,10 +627,7 @@ def summarize_v2_task(
|
||||
1. Run summary internals
|
||||
2. Sends the final summary via webhook.
|
||||
"""
|
||||
try:
|
||||
payload = SummarizeTaskJob.model_validate(payload)
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
payload = SummarizeTaskJob.model_validate(payload)
|
||||
summary = summarize_transcription_internals(
|
||||
distinct_id=payload.user_sub,
|
||||
transcript=payload.content,
|
||||
@@ -713,22 +652,18 @@ def summarize_v2_task(
|
||||
summary_data_url=file_service.get_summary_signed_url(job_id),
|
||||
)
|
||||
call_webhook_v2_task.apply_async(
|
||||
args=[dump_for_transport(success_payload), payload.tenant_id]
|
||||
args=[success_payload.model_dump(), payload.tenant_id]
|
||||
)
|
||||
metadata_manager.capture(job_id, settings.posthog_summary_success)
|
||||
|
||||
return dump_for_transport(success_payload)
|
||||
return success_payload.model_dump()
|
||||
|
||||
|
||||
@signals.task_prerun.connect(sender=summarize_v2_task)
|
||||
def task_started_summary(task_id=None, task=None, args=None, **kwargs):
|
||||
"""Signal handler called before task execution begins."""
|
||||
if args:
|
||||
try:
|
||||
payload = SummarizeTaskJob.model_validate(args[0])
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
metadata_manager.create(task_id, payload)
|
||||
metadata_manager.create(task_id, SummarizeTaskJob.model_validate(args[0]))
|
||||
|
||||
|
||||
@signals.task_retry.connect(sender=summarize_v2_task)
|
||||
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
import logging
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from requests import RequestException, Session
|
||||
from requests import Session
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util import Retry
|
||||
|
||||
@@ -43,8 +43,6 @@ def _post_with_retries(*, url, data, api_key: str | None = None):
|
||||
response = session.post(url, json=data, timeout=(20, 3 * 60))
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except RequestException:
|
||||
raise RequestException("Docs delivery failed") from None
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
"""Validation errors suitable for logging and transport."""
|
||||
|
||||
from typing import get_args
|
||||
|
||||
from pydantic import SecretStr, ValidationError
|
||||
from pydantic_core import ErrorType, PydanticCustomError
|
||||
|
||||
_BUILTIN_ERROR_TYPES = frozenset(get_args(ErrorType))
|
||||
|
||||
|
||||
def clean_validation_errors(errors: list[dict]) -> list[dict]:
|
||||
"""Mask SecretStr values, including those nested in validation input."""
|
||||
|
||||
def clean(value):
|
||||
if isinstance(value, SecretStr):
|
||||
return "[redacted]"
|
||||
if isinstance(value, Exception):
|
||||
# Do not retain validator exception objects, whose args may hold secrets.
|
||||
return str(value)
|
||||
if isinstance(value, dict):
|
||||
return {key: clean(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [clean(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(clean(item) for item in value)
|
||||
return value
|
||||
|
||||
return clean(errors)
|
||||
|
||||
|
||||
class CleanedValidationError(ValidationError):
|
||||
"""A new Pydantic exception with secret inputs replaced by a marker."""
|
||||
|
||||
def __init__(self, error, *args, secret_input=False, **kwargs):
|
||||
"""State is initialized by Pydantic during __new__."""
|
||||
|
||||
def __new__(cls, error, *args, secret_input=False, **kwargs):
|
||||
"""Accept an original error, or Pydantic's native construction arguments."""
|
||||
if not isinstance(error, ValidationError):
|
||||
# Pydantic uses this path when constructing or unpickling the error.
|
||||
return super().__new__(cls, error, *args, **kwargs)
|
||||
|
||||
details = clean_validation_errors(error.errors(include_url=False))
|
||||
for detail in details:
|
||||
if secret_input:
|
||||
# An adapter may have received the unwrapped value of a SecretStr.
|
||||
detail["input"] = "[redacted]"
|
||||
return cls._from_details(error.title, details)
|
||||
|
||||
@classmethod
|
||||
def _from_details(cls, title, details):
|
||||
"""Reconstruct builtin and custom Pydantic error codes."""
|
||||
for detail in details:
|
||||
if detail["type"] not in _BUILTIN_ERROR_TYPES:
|
||||
detail["type"] = PydanticCustomError(
|
||||
detail["type"], detail["msg"], detail.get("ctx")
|
||||
)
|
||||
return cls.from_exception_data(title, details, hide_input=True)
|
||||
|
||||
def __reduce__(self):
|
||||
"""Preserve custom error codes when Celery pickles the exception."""
|
||||
return self._from_details, (self.title, self.errors(include_url=False))
|
||||
@@ -17,7 +17,6 @@ import requests
|
||||
from botocore.config import Config
|
||||
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.privacy import redact_url
|
||||
from summary.core.shared_models import WhisperXResponse
|
||||
|
||||
settings = get_settings()
|
||||
@@ -315,7 +314,7 @@ class FileService:
|
||||
"""Download file from a cloud storage URL to local temporary file."""
|
||||
logger.info(
|
||||
"Download recording from URL | cloud_storage_url: %s",
|
||||
redact_url(cloud_storage_url),
|
||||
cloud_storage_url,
|
||||
)
|
||||
|
||||
if not cloud_storage_url:
|
||||
@@ -353,10 +352,10 @@ class FileService:
|
||||
logger.debug("Recording local file path: %s", local_path)
|
||||
return local_path
|
||||
|
||||
except requests.RequestException:
|
||||
except requests.RequestException as e:
|
||||
raise FileServiceException(
|
||||
"Unexpected error while downloading object from cloud_storage_url."
|
||||
) from None
|
||||
) from e
|
||||
|
||||
def _validate_duration(self, duration_seconds: float) -> None:
|
||||
"""Validate audio file duration against configured maximum."""
|
||||
@@ -377,7 +376,7 @@ class FileService:
|
||||
|
||||
def read_cloud_storage_json(self, cloud_storage_url: str) -> dict:
|
||||
"""Read and parse a JSON file from a url."""
|
||||
logger.info("Reading JSON: %s", redact_url(cloud_storage_url))
|
||||
logger.info("Reading JSON: %s", cloud_storage_url)
|
||||
res = requests.get(cloud_storage_url, timeout=(10, 100))
|
||||
res.raise_for_status()
|
||||
return res.json()
|
||||
|
||||
@@ -140,6 +140,6 @@ class LLMService:
|
||||
response = self._client.chat.completions.create(**params)
|
||||
return response.choices[0].message.content
|
||||
|
||||
except Exception:
|
||||
logger.error("LLM call failed")
|
||||
raise LLMException("LLM call failed") from None
|
||||
except Exception as e:
|
||||
logger.exception("LLM call failed: %s", e)
|
||||
raise LLMException(f"LLM call failed: {e}") from e
|
||||
|
||||
@@ -2,17 +2,10 @@
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import (
|
||||
AwareDatetime,
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
EmailStr,
|
||||
Field,
|
||||
field_validator,
|
||||
)
|
||||
from pydantic import AwareDatetime, BaseModel, EmailStr, Field, field_validator
|
||||
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.types import SecretUrl
|
||||
from summary.core.types import Url
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
@@ -20,8 +13,6 @@ settings = get_settings()
|
||||
class SharedV2TaskCreation(BaseModel):
|
||||
"""Model that holds basic information for task creation."""
|
||||
|
||||
model_config = ConfigDict(hide_input_in_errors=True)
|
||||
|
||||
user_sub: str = Field(title="User Sub", description="The user's sub.")
|
||||
user_email: EmailStr | None = Field(
|
||||
default=None,
|
||||
@@ -33,9 +24,7 @@ class SharedV2TaskCreation(BaseModel):
|
||||
class RecordingMetadata(BaseModel):
|
||||
"""Model for recording metadata."""
|
||||
|
||||
model_config = ConfigDict(hide_input_in_errors=True)
|
||||
|
||||
cloud_storage_url: SecretUrl = Field(
|
||||
cloud_storage_url: Url = Field(
|
||||
title="Cloud Storage URL",
|
||||
description="The URL of the metadata file for speaker assignment.",
|
||||
)
|
||||
@@ -46,8 +35,6 @@ class RecordingMetadata(BaseModel):
|
||||
class PushToDocsBaseConfig(BaseModel):
|
||||
"""Model containing information for pushing transcript and summaries to docs."""
|
||||
|
||||
model_config = ConfigDict(hide_input_in_errors=True)
|
||||
|
||||
user_email: EmailStr = Field(
|
||||
title="User Email", description="The user's email, future owner of the docs."
|
||||
)
|
||||
@@ -57,12 +44,12 @@ class PushToDocsBaseConfig(BaseModel):
|
||||
class PushToDocsTranscriptConfig(PushToDocsBaseConfig):
|
||||
"""Model for push to docs information for transcripts."""
|
||||
|
||||
download_link: SecretUrl | None = Field(
|
||||
download_link: Url | None = Field(
|
||||
default=None,
|
||||
title="Download Link",
|
||||
description="The link to download the recording.",
|
||||
)
|
||||
form_link: SecretUrl | None = Field(
|
||||
form_link: Url | None = Field(
|
||||
default=None,
|
||||
title="Form Link",
|
||||
description="The link to fill out a form for the recording.",
|
||||
@@ -82,7 +69,7 @@ class PushToDocsSummaryConfig(PushToDocsBaseConfig):
|
||||
class TranscribeTaskApiRequest(SharedV2TaskCreation):
|
||||
"""Model for creating a transcribe task (used for API request)."""
|
||||
|
||||
cloud_storage_url: SecretUrl = Field(
|
||||
cloud_storage_url: Url = Field(
|
||||
title="Cloud storage URL",
|
||||
description="The URL of the audio file to transcribe.",
|
||||
)
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Privacy helpers for observability."""
|
||||
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from pydantic import BaseModel, SecretStr
|
||||
|
||||
|
||||
def dump_for_transport(model: BaseModel) -> dict:
|
||||
"""Reveal secrets explicitly for task messages, HTTP bodies and API responses."""
|
||||
|
||||
def reveal(value):
|
||||
if isinstance(value, SecretStr):
|
||||
return value.get_secret_value()
|
||||
if isinstance(value, dict):
|
||||
return {key: reveal(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [reveal(item) for item in value]
|
||||
return value
|
||||
|
||||
return reveal(model.model_dump())
|
||||
|
||||
|
||||
def redact_url(url):
|
||||
"""Keep the source identifiable without logging query or user credentials."""
|
||||
if not isinstance(url, str):
|
||||
return "[redacted]"
|
||||
# Remove query and fragment before parsing, including for malformed URLs.
|
||||
base, has_query, _ = url.partition("?")
|
||||
base = base.partition("#")[0]
|
||||
try:
|
||||
parts = urlsplit(base)
|
||||
except ValueError:
|
||||
return "[redacted]"
|
||||
authority = parts.netloc.rsplit("@", 1)[-1]
|
||||
redacted = urlunsplit(parts._replace(netloc=authority))
|
||||
return redacted + ("?[redacted]" if has_query else "")
|
||||
@@ -1,12 +1,9 @@
|
||||
"""Sentry configuration."""
|
||||
|
||||
import re
|
||||
|
||||
import sentry_sdk
|
||||
from sentry_sdk.scrubber import DEFAULT_DENYLIST, DEFAULT_PII_DENYLIST, EventScrubber
|
||||
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.privacy import redact_url
|
||||
|
||||
# Exact names (case-insensitive) of variables and dict keys to redact.
|
||||
SENSITIVE_DATA_DENYLIST = [
|
||||
@@ -93,45 +90,5 @@ def init_sentry() -> None:
|
||||
# Task creation requests carry the content to summarize.
|
||||
max_request_body_size="never",
|
||||
include_local_variables=True,
|
||||
before_send=before_send,
|
||||
before_send_transaction=before_send,
|
||||
before_breadcrumb=before_breadcrumb,
|
||||
event_scrubber=build_event_scrubber(),
|
||||
)
|
||||
|
||||
|
||||
def _scrub_strings(value):
|
||||
"""Remove URL credentials, queries and fragments throughout an event."""
|
||||
if isinstance(value, str):
|
||||
return re.sub(
|
||||
r"https?://[^\s\"'<>]+",
|
||||
lambda m: redact_url(m.group()),
|
||||
value,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
if isinstance(value, dict):
|
||||
return {key: _scrub_strings(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_scrub_strings(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def before_send(event, hint):
|
||||
"""Keep exception types and stacks, but discard untrusted message text."""
|
||||
del hint
|
||||
event.pop("message", None)
|
||||
event.pop("logentry", None)
|
||||
request = event.get("request", {})
|
||||
request.pop("data", None)
|
||||
request.pop("query_string", None)
|
||||
for exception in event.get("exception", {}).get("values", []):
|
||||
exception["value"] = "[Filtered]"
|
||||
return _scrub_strings(event)
|
||||
|
||||
|
||||
def before_breadcrumb(crumb, hint):
|
||||
"""Drop arbitrary breadcrumb text and scrub URLs in structured data."""
|
||||
del hint
|
||||
crumb.pop("message", None)
|
||||
crumb.pop("data", None)
|
||||
return _scrub_strings(crumb)
|
||||
|
||||
@@ -2,9 +2,7 @@
|
||||
|
||||
from typing import Annotated, Literal, Union
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
||||
|
||||
from summary.core.types import SecretUrl
|
||||
from pydantic import BaseModel, Field, TypeAdapter
|
||||
|
||||
|
||||
class WordSegment(BaseModel):
|
||||
@@ -67,8 +65,6 @@ class WhisperXResponse(BaseModel):
|
||||
class BaseWebhook(BaseModel):
|
||||
"""Base webhook payload."""
|
||||
|
||||
model_config = ConfigDict(hide_input_in_errors=True)
|
||||
|
||||
job_id: str = Field(
|
||||
title="Job ID",
|
||||
description="The ID of the job document in the receiver system.",
|
||||
@@ -80,7 +76,7 @@ class TranscribeWebhookSuccessPayload(BaseWebhook):
|
||||
|
||||
type: Literal["transcript"] = Field(default="transcript")
|
||||
status: Literal["success"] = Field(default="success")
|
||||
transcription_data_url: SecretUrl = Field(
|
||||
transcription_data_url: str = Field(
|
||||
title="Transcript", description="URL to the raw transcription data."
|
||||
)
|
||||
|
||||
@@ -125,7 +121,7 @@ class SummarizeWebhookSuccessPayload(BaseWebhook):
|
||||
|
||||
type: Literal["summary"] = Field(default="summary")
|
||||
status: Literal["success"] = Field(default="success")
|
||||
summary_data_url: SecretUrl = Field(
|
||||
summary_data_url: str = Field(
|
||||
title="Summary", description="URL to the raw summary data."
|
||||
)
|
||||
|
||||
|
||||
@@ -2,31 +2,9 @@
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import (
|
||||
AfterValidator,
|
||||
BeforeValidator,
|
||||
HttpUrl,
|
||||
SecretStr,
|
||||
TypeAdapter,
|
||||
ValidationError,
|
||||
)
|
||||
|
||||
from summary.core.exceptions import CleanedValidationError
|
||||
from pydantic import BeforeValidator, HttpUrl, TypeAdapter
|
||||
|
||||
http_url_adapter = TypeAdapter(HttpUrl)
|
||||
Url = Annotated[
|
||||
str, BeforeValidator(lambda value: str(http_url_adapter.validate_python(value)))
|
||||
]
|
||||
|
||||
|
||||
def _validate_secret_url(value: SecretStr) -> SecretStr:
|
||||
"""Validate and normalize a URL without echoing it in validator messages."""
|
||||
try:
|
||||
return SecretStr(
|
||||
str(http_url_adapter.validate_python(value.get_secret_value()))
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise CleanedValidationError(exc, secret_input=True) from None
|
||||
|
||||
|
||||
SecretUrl = Annotated[SecretStr, AfterValidator(_validate_secret_url)]
|
||||
|
||||
@@ -6,7 +6,6 @@ import logging
|
||||
import requests
|
||||
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.privacy import dump_for_transport
|
||||
from summary.core.shared_models import (
|
||||
WebhookPayloads,
|
||||
)
|
||||
@@ -30,19 +29,16 @@ def call_webhook_v2(
|
||||
logger.debug("Submitting to %s", tenant.webhook_url)
|
||||
logger.debug("Request payload: %s", payload.model_dump_json(indent=2))
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
tenant.webhook_url,
|
||||
json=dump_for_transport(payload),
|
||||
headers={
|
||||
"Authorization": f"Bearer {tenant.webhook_api_key.get_secret_value()}",
|
||||
"User-Agent": settings.app_external_user_agent,
|
||||
},
|
||||
timeout=(10, 20),
|
||||
)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException:
|
||||
raise requests.RequestException("Webhook delivery failed") from None
|
||||
response = requests.post(
|
||||
tenant.webhook_url,
|
||||
json=payload.model_dump(),
|
||||
headers={
|
||||
"Authorization": f"Bearer {tenant.webhook_api_key.get_secret_value()}",
|
||||
"User-Agent": settings.app_external_user_agent,
|
||||
},
|
||||
timeout=(10, 20),
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
try:
|
||||
response_data = response.json()
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
"""Application."""
|
||||
|
||||
from dockerflow.fastapi import router as dockerflow_router
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi import FastAPI
|
||||
|
||||
from summary.api.main import api_router_v2
|
||||
from summary.core import checks # noqa: F401 -- registers the Dockerflow checks
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.exceptions import clean_validation_errors
|
||||
from summary.core.sentry import init_sentry
|
||||
|
||||
settings = get_settings()
|
||||
@@ -23,13 +19,3 @@ app = FastAPI(
|
||||
|
||||
app.include_router(api_router_v2, prefix=settings.app_api_v2_str)
|
||||
app.include_router(dockerflow_router)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(request: Request, exc: RequestValidationError):
|
||||
"""Keep validation details while masking SecretStr input values."""
|
||||
del request
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content=jsonable_encoder({"detail": clean_validation_errors(exc.errors())}),
|
||||
)
|
||||
|
||||
@@ -3,10 +3,6 @@
|
||||
from datetime import datetime
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from summary.core.exceptions import CleanedValidationError
|
||||
|
||||
|
||||
class TestTasksV2:
|
||||
"""Tests for the /v2/async-jobs-jobs endpoints."""
|
||||
@@ -178,163 +174,3 @@ class TestTasksV2:
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json() == {"detail": "Forbidden"}
|
||||
|
||||
|
||||
def test_validation_preserves_ordinary_input(client):
|
||||
"""Validation errors keep input values that are not SecretStr."""
|
||||
canary = "private-meeting-content"
|
||||
response = client.post(
|
||||
"/api/v2/async-jobs/summarize",
|
||||
headers={"Authorization": "Bearer test-api-token"},
|
||||
json={"user_sub": "user", "content": {"unexpected": canary}},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert canary in response.text
|
||||
assert response.json()["detail"] == [
|
||||
{
|
||||
"type": "string_type",
|
||||
"loc": ["body", "content"],
|
||||
"msg": "Input should be a valid string",
|
||||
"input": {"unexpected": canary},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_validation_preserves_context(client):
|
||||
"""Validator errors retain location, message and serializable context."""
|
||||
canary = "private-invalid-url-signature"
|
||||
response = client.post(
|
||||
"/api/v2/async-jobs/transcribe",
|
||||
headers={"Authorization": "Bearer test-api-token"},
|
||||
json={
|
||||
"user_sub": "user",
|
||||
"language": "en",
|
||||
"cloud_storage_url": f"invalid-url?signature={canary}",
|
||||
},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert canary not in response.text
|
||||
assert response.json()["detail"] == [
|
||||
{
|
||||
"type": "url_parsing",
|
||||
"loc": ["body", "cloud_storage_url"],
|
||||
"msg": "Input should be a valid URL, relative URL without a base",
|
||||
"input": "[redacted]",
|
||||
"ctx": {"error": "relative URL without a base"},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@patch(
|
||||
"summary.api.route.tasks_v2.process_audio_transcribe_v2_task.apply_async",
|
||||
return_value=MagicMock(id="transcribe-job"),
|
||||
)
|
||||
def test_signed_urls_are_unwrapped_for_task_submission(mock_apply_async, client):
|
||||
"""API input secrets reach Celery intact, including nested metadata URLs."""
|
||||
signed_url = "https://storage.example.com/file?signature=deterministic-signature"
|
||||
response = client.post(
|
||||
"/api/v2/async-jobs/transcribe",
|
||||
headers={"Authorization": "Bearer test-api-token"},
|
||||
json={
|
||||
"user_sub": "user",
|
||||
"language": "en",
|
||||
"cloud_storage_url": signed_url,
|
||||
"metadata": {
|
||||
"cloud_storage_url": signed_url,
|
||||
"started_at": "2026-01-01T10:00:00Z",
|
||||
"ended_at": "2026-01-01T11:00:00Z",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
payload = mock_apply_async.call_args.kwargs["args"][0]
|
||||
assert payload["cloud_storage_url"] == signed_url
|
||||
assert payload["metadata"]["cloud_storage_url"] == signed_url
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
("transcribe", "transcript", "transcription_data_url"),
|
||||
("summarize", "summary", "summary_data_url"),
|
||||
],
|
||||
)
|
||||
@patch("summary.api.route.tasks_v2.celery")
|
||||
@patch("summary.api.route.tasks_v2.AsyncResult")
|
||||
def test_success_status_returns_usable_signed_url(
|
||||
mock_async_result, mock_celery, client, case
|
||||
):
|
||||
"""Success responses explicitly reveal result URLs for authorized clients."""
|
||||
job_type, payload_type, url_field = case
|
||||
signed_url = "https://storage.example.com/result?signature=deterministic-signature"
|
||||
mock_celery.backend.client.exists.return_value = True
|
||||
mock_async_result.return_value = MagicMock(
|
||||
status="SUCCESS",
|
||||
args=[{"tenant_id": "test-tenant"}],
|
||||
result={"job_id": "job", url_field: signed_url},
|
||||
)
|
||||
response = client.get(
|
||||
f"/api/v2/async-jobs/{job_type}/job",
|
||||
headers={"Authorization": "Bearer test-api-token"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"job_id": "job",
|
||||
"type": payload_type,
|
||||
"status": "success",
|
||||
url_field: signed_url,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
("transcribe", "transcription_data_url"),
|
||||
("summarize", "summary_data_url"),
|
||||
],
|
||||
)
|
||||
@patch("summary.api.route.tasks_v2.celery")
|
||||
@patch("summary.api.route.tasks_v2.AsyncResult")
|
||||
def test_invalid_success_result_hides_revealed_url(
|
||||
mock_async_result, mock_celery, client, case
|
||||
):
|
||||
"""Missing result fields cannot retain a raw signed URL in error details."""
|
||||
job_type, url_field = case
|
||||
canary = "private-result-signature"
|
||||
mock_celery.backend.client.exists.return_value = True
|
||||
mock_async_result.return_value = MagicMock(
|
||||
status="SUCCESS",
|
||||
args=[{"tenant_id": "test-tenant"}],
|
||||
result={url_field: f"https://storage.example.com/result?signature={canary}"},
|
||||
)
|
||||
with pytest.raises(CleanedValidationError) as exc:
|
||||
client.get(
|
||||
f"/api/v2/async-jobs/{job_type}/job",
|
||||
headers={"Authorization": "Bearer test-api-token"},
|
||||
)
|
||||
assert exc.value.errors()[0]["type"] == "missing"
|
||||
assert exc.value.errors()[0]["loc"] == ("job_id",)
|
||||
assert exc.value.errors()[0]["input"] == "[redacted]"
|
||||
assert canary not in str(exc.value)
|
||||
assert canary not in exc.value.json()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nested_error", [False, True])
|
||||
def test_missing_field_errors_preserve_raw_json_strings(client, nested_error):
|
||||
"""Raw JSON input is retained when it has not been converted to SecretStr."""
|
||||
canary = "private-validation-signature"
|
||||
signed_url = f"https://storage.example.com/recording?signature={canary}"
|
||||
payload = {"language": "en", "cloud_storage_url": signed_url}
|
||||
if nested_error:
|
||||
payload["user_sub"] = "user"
|
||||
payload["metadata"] = {"cloud_storage_url": signed_url}
|
||||
response = client.post(
|
||||
"/api/v2/async-jobs/transcribe",
|
||||
headers={"Authorization": "Bearer test-api-token"},
|
||||
json=payload,
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert canary in response.text
|
||||
assert all(error["type"] == "missing" for error in response.json()["detail"])
|
||||
expected_input = payload["metadata"] if nested_error else payload
|
||||
assert all(error["input"] == expected_input for error in response.json()["detail"])
|
||||
|
||||
@@ -22,12 +22,6 @@ def get_settings_override():
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def settings_override():
|
||||
"""Provide isolated settings containing the test tenant."""
|
||||
return get_settings_override()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client():
|
||||
"""Provide a FastAPI TestClient for tests."""
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
"""Secret URLs stay masked until explicitly serialized for transport."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from kombu.serialization import dumps, loads
|
||||
from pydantic import SecretStr, ValidationError
|
||||
|
||||
from summary.core import celery_worker
|
||||
from summary.core.models import (
|
||||
PushToDocsTranscriptConfig,
|
||||
RecordingMetadata,
|
||||
TranscribeTaskApiRequest,
|
||||
TranscribeTaskJob,
|
||||
)
|
||||
from summary.core.privacy import dump_for_transport
|
||||
from summary.core.shared_models import (
|
||||
SummarizeWebhookSuccessPayload,
|
||||
TranscribeWebhookSuccessPayload,
|
||||
WhisperXResponse,
|
||||
)
|
||||
|
||||
CANARY = "private-signed-url-canary"
|
||||
SIGNED_URL = f"https://storage.example.com/recording.mp4?signature={CANARY}"
|
||||
METADATA = {
|
||||
"cloud_storage_url": SIGNED_URL,
|
||||
"started_at": "2026-01-01T10:00:00Z",
|
||||
"ended_at": "2026-01-01T11:00:00Z",
|
||||
}
|
||||
DOCS_CONFIG = {
|
||||
"user_email": "user@example.com",
|
||||
"title": "Meeting",
|
||||
"download_link": SIGNED_URL,
|
||||
"form_link": SIGNED_URL,
|
||||
}
|
||||
MODEL_CASES = [
|
||||
(RecordingMetadata, METADATA, "cloud_storage_url"),
|
||||
(
|
||||
TranscribeTaskApiRequest,
|
||||
{"user_sub": "user", "language": "en", "cloud_storage_url": SIGNED_URL},
|
||||
"cloud_storage_url",
|
||||
),
|
||||
(PushToDocsTranscriptConfig, DOCS_CONFIG, "download_link"),
|
||||
(PushToDocsTranscriptConfig, DOCS_CONFIG, "form_link"),
|
||||
(
|
||||
TranscribeWebhookSuccessPayload,
|
||||
{"job_id": "job", "transcription_data_url": SIGNED_URL},
|
||||
"transcription_data_url",
|
||||
),
|
||||
(
|
||||
SummarizeWebhookSuccessPayload,
|
||||
{"job_id": "job", "summary_data_url": SIGNED_URL},
|
||||
"summary_data_url",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "data", "field"), MODEL_CASES)
|
||||
def test_signed_urls_are_masked_by_default(model, data, field):
|
||||
"""Reprs, ordinary dumps and JSON output mask all signed URL fields."""
|
||||
value = model.model_validate(data)
|
||||
secret = getattr(value, field)
|
||||
assert isinstance(secret, SecretStr)
|
||||
assert secret.get_secret_value() == SIGNED_URL
|
||||
assert CANARY not in repr(value)
|
||||
assert CANARY not in str(value.model_dump())
|
||||
assert CANARY not in value.model_dump_json()
|
||||
assert dump_for_transport(value)[field] == SIGNED_URL
|
||||
assert getattr(model.model_validate(value.model_dump()), field) == secret
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "data", "field"), MODEL_CASES)
|
||||
def test_missing_fields_do_not_echo_signed_urls(model, data, field):
|
||||
"""Root validation errors omit raw input even before secrets are wrapped."""
|
||||
required_field = (
|
||||
"job_id"
|
||||
if "job_id" in data
|
||||
else next(
|
||||
key for key in data if key not in {field, "download_link", "form_link"}
|
||||
)
|
||||
)
|
||||
invalid_data = {key: value for key, value in data.items() if key != required_field}
|
||||
with pytest.raises(ValidationError) as exc:
|
||||
model.model_validate(invalid_data)
|
||||
assert CANARY not in str(exc.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "data", "field"), MODEL_CASES[:4])
|
||||
def test_invalid_signed_urls_have_sanitized_validation_messages(model, data, field):
|
||||
"""URL validation still rejects invalid URLs without quoting their value."""
|
||||
with pytest.raises(ValidationError) as exc:
|
||||
model.model_validate({**data, field: f"not-a-url?signature={CANARY}"})
|
||||
assert CANARY not in str(exc.value)
|
||||
assert CANARY not in str(
|
||||
exc.value.errors(include_input=False, include_context=False)
|
||||
)
|
||||
assert exc.value.errors()[0]["type"] == "url_parsing"
|
||||
assert exc.value.errors()[0]["input"] == "[redacted]"
|
||||
|
||||
|
||||
def test_nested_task_urls_survive_broker_serialization():
|
||||
"""Nested secrets round-trip through Celery's JSON serializer as real URLs."""
|
||||
payload = TranscribeTaskJob(
|
||||
user_sub="user",
|
||||
language="en",
|
||||
cloud_storage_url=SIGNED_URL,
|
||||
tenant_id="tenant",
|
||||
received_at="2026-01-01T10:00:00Z",
|
||||
metadata=METADATA,
|
||||
push_to_docs_config=DOCS_CONFIG,
|
||||
)
|
||||
assert CANARY not in repr(payload)
|
||||
assert CANARY not in payload.model_dump_json()
|
||||
content_type, encoding, body = dumps(dump_for_transport(payload), serializer="json")
|
||||
restored = TranscribeTaskJob.model_validate(loads(body, content_type, encoding))
|
||||
assert restored.cloud_storage_url.get_secret_value() == SIGNED_URL
|
||||
assert restored.metadata.cloud_storage_url.get_secret_value() == SIGNED_URL
|
||||
assert restored.push_to_docs_config.download_link.get_secret_value() == SIGNED_URL
|
||||
assert restored.push_to_docs_config.form_link.get_secret_value() == SIGNED_URL
|
||||
|
||||
|
||||
def test_transcription_worker_uses_real_urls(monkeypatch):
|
||||
"""The worker unwraps input URLs and publishes usable signed result URLs."""
|
||||
transcription = WhisperXResponse(segments=())
|
||||
transcribe = Mock(return_value=transcription)
|
||||
monkeypatch.setattr(celery_worker, "transcribe_audio", transcribe)
|
||||
monkeypatch.setattr(celery_worker, "_should_push_to_docs", Mock(return_value=True))
|
||||
monkeypatch.setattr(
|
||||
celery_worker, "_should_auto_create_summary", Mock(return_value=False)
|
||||
)
|
||||
formatter = Mock(return_value="Formatted transcript")
|
||||
monkeypatch.setattr(celery_worker, "format_transcript", formatter)
|
||||
monkeypatch.setattr(celery_worker, "create_document_in_lasuite_docs", Mock())
|
||||
storage = Mock()
|
||||
storage.get_transcript_signed_url.return_value = SIGNED_URL
|
||||
monkeypatch.setattr(celery_worker, "file_service", storage)
|
||||
monkeypatch.setattr(celery_worker, "metadata_manager", Mock())
|
||||
publish = Mock()
|
||||
monkeypatch.setattr(celery_worker.call_webhook_v2_task, "apply_async", publish)
|
||||
task = celery_worker.process_audio_transcribe_v2_task
|
||||
monkeypatch.setattr(task.request, "id", "job")
|
||||
|
||||
result = task.run(
|
||||
{
|
||||
"user_sub": "user",
|
||||
"language": "en",
|
||||
"cloud_storage_url": SIGNED_URL,
|
||||
"tenant_id": "tenant",
|
||||
"received_at": "2026-01-01T10:00:00Z",
|
||||
"push_to_docs_config": DOCS_CONFIG,
|
||||
}
|
||||
)
|
||||
assert transcribe.call_args.kwargs["cloud_storage_url"] == SIGNED_URL
|
||||
assert formatter.call_args.args[3:] == (SIGNED_URL, SIGNED_URL)
|
||||
assert result["transcription_data_url"] == SIGNED_URL
|
||||
assert publish.call_args.kwargs["args"][0] == result
|
||||
|
||||
|
||||
def test_recording_metadata_url_is_unwrapped(monkeypatch):
|
||||
"""Speaker assignment fetches metadata using the real URL."""
|
||||
storage = Mock()
|
||||
monkeypatch.setattr(celery_worker, "file_service", storage)
|
||||
resolver = Mock()
|
||||
resolver.return_value.apply_to.return_value = {"segments": []}
|
||||
monkeypatch.setattr(celery_worker, "resolve_speaker_identities", resolver)
|
||||
celery_worker.resolve_speaker_identities_and_apply_to(
|
||||
transcription=WhisperXResponse(segments=()),
|
||||
recording_metadata=RecordingMetadata.model_validate(METADATA),
|
||||
task_id="job",
|
||||
)
|
||||
storage.read_cloud_storage_json.assert_called_once_with(SIGNED_URL)
|
||||
@@ -1,209 +0,0 @@
|
||||
"""Regression tests for observability privacy boundaries."""
|
||||
|
||||
import logging
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from summary.core import analytics, celery_worker, webhook_service
|
||||
from summary.core.llm_service import LLMException, LLMService
|
||||
from summary.core.privacy import redact_url
|
||||
from summary.core.sentry import before_breadcrumb, before_send
|
||||
from summary.core.shared_models import SummarizeWebhookSuccessPayload
|
||||
|
||||
CANARY = "sensitive-canary"
|
||||
URL = f"https://user:password@example.com/file?signature={CANARY}#fragment"
|
||||
|
||||
|
||||
def test_url_redaction():
|
||||
"""Credentials and signed query strings are removed."""
|
||||
assert redact_url(URL) == "https://example.com/file?[redacted]"
|
||||
|
||||
|
||||
def test_sentry_scrubs_events_and_breadcrumbs():
|
||||
"""Untrusted exception messages and breadcrumb data never leave in events."""
|
||||
event = {
|
||||
"request": {"url": URL, "data": CANARY, "query_string": CANARY},
|
||||
"exception": {
|
||||
"values": [
|
||||
{
|
||||
"type": "ValueError",
|
||||
"value": CANARY,
|
||||
"stacktrace": {"frames": [{"vars": {"job_id": "job-1"}}]},
|
||||
}
|
||||
]
|
||||
},
|
||||
"extra": {"url": URL},
|
||||
}
|
||||
result = before_send(event, {})
|
||||
assert CANARY not in str(result)
|
||||
assert result["exception"]["values"][0]["type"] == "ValueError"
|
||||
assert result["exception"]["values"][0]["stacktrace"]["frames"][0]["vars"] == {
|
||||
"job_id": "job-1"
|
||||
}
|
||||
assert before_breadcrumb({"message": CANARY, "data": {"url": URL}}, {}) == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheme", ["http", "HTTP", "hTtP", "https", "HTTPS", "hTtPs"])
|
||||
def test_sentry_url_scrubbing_is_case_insensitive(scheme):
|
||||
"""Scheme casing cannot bypass redaction of URLs nested in event strings."""
|
||||
event = {
|
||||
"extra": {
|
||||
"nested": [
|
||||
{
|
||||
"detail": (
|
||||
f"prefix <{scheme}://user:password@example.com/Path"
|
||||
f"?signature={CANARY}#fragment> suffix"
|
||||
)
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
result = before_send(event, {})
|
||||
assert result["extra"]["nested"][0]["detail"] == (
|
||||
f"prefix <{scheme.lower()}://example.com/Path?[redacted]> suffix"
|
||||
)
|
||||
assert CANARY not in str(result)
|
||||
|
||||
|
||||
def test_llm_exception_does_not_include_upstream_message(caplog):
|
||||
"""LLM failures retain their category without logging upstream text."""
|
||||
observability = Mock(is_enabled=False)
|
||||
observability.get_openai_client.return_value.chat.completions.create.side_effect = (
|
||||
ValueError(CANARY)
|
||||
)
|
||||
with caplog.at_level(logging.DEBUG), pytest.raises(LLMException) as exc:
|
||||
LLMService(observability).call("system", CANARY, "test")
|
||||
assert CANARY not in str(exc.value)
|
||||
assert CANARY not in caplog.text
|
||||
assert exc.value.__suppress_context__
|
||||
|
||||
|
||||
@pytest.mark.parametrize("log_level", [logging.DEBUG, logging.INFO])
|
||||
def test_webhook_content_is_only_logged_at_debug(
|
||||
monkeypatch, caplog, settings_override, log_level
|
||||
):
|
||||
"""DEBUG retains response content; INFO and above only contain metadata."""
|
||||
monkeypatch.setattr(webhook_service, "settings", settings_override)
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"id": "document-123", "content": CANARY}
|
||||
monkeypatch.setattr(webhook_service.requests, "post", Mock(return_value=response))
|
||||
payload = SummarizeWebhookSuccessPayload(job_id="job", summary_data_url=URL)
|
||||
with caplog.at_level(log_level):
|
||||
webhook_service.call_webhook_v2(tenant_id="test-tenant", payload=payload)
|
||||
assert (
|
||||
webhook_service.requests.post.call_args.kwargs["json"]["summary_data_url"]
|
||||
== URL
|
||||
)
|
||||
assert all(
|
||||
CANARY not in record.getMessage()
|
||||
for record in caplog.records
|
||||
if record.levelno >= logging.INFO
|
||||
)
|
||||
if log_level == logging.DEBUG:
|
||||
assert any(
|
||||
record.levelno == logging.DEBUG and CANARY in record.getMessage()
|
||||
for record in caplog.records
|
||||
)
|
||||
else:
|
||||
assert CANARY not in caplog.text
|
||||
assert "Document document-123 submitted (HTTP 200)" in caplog.text
|
||||
|
||||
|
||||
def test_webhook_failure_keeps_retry_category(monkeypatch, settings_override):
|
||||
"""Webhook exceptions remain retryable without upstream text."""
|
||||
monkeypatch.setattr(webhook_service, "settings", settings_override)
|
||||
monkeypatch.setattr(
|
||||
webhook_service.requests,
|
||||
"post",
|
||||
Mock(side_effect=requests.RequestException(URL)),
|
||||
)
|
||||
payload = SummarizeWebhookSuccessPayload(job_id="job", summary_data_url=URL)
|
||||
with pytest.raises(requests.RequestException) as exc:
|
||||
webhook_service.call_webhook_v2(tenant_id="test-tenant", payload=payload)
|
||||
assert CANARY not in str(exc.value)
|
||||
assert exc.value.__suppress_context__
|
||||
|
||||
|
||||
def test_analytics_failure_does_not_chain_payload(monkeypatch):
|
||||
"""Analytics errors do not expose arbitrary client exception text."""
|
||||
client = analytics.Analytics()
|
||||
client._client = Mock()
|
||||
client._client.capture.side_effect = ValueError(CANARY)
|
||||
with pytest.raises(analytics.AnalyticsException) as exc:
|
||||
client.capture("event", "user")
|
||||
assert CANARY not in str(exc.value)
|
||||
assert exc.value.__suppress_context__
|
||||
|
||||
|
||||
@pytest.mark.parametrize("log_level", [logging.DEBUG, logging.INFO, logging.ERROR])
|
||||
def test_celery_log_filter_respects_log_level(log_level):
|
||||
"""Celery trace values remain available in DEBUG only."""
|
||||
record = logging.LogRecord(
|
||||
"celery.app.trace", log_level, "", 0, "%s", (CANARY,), None
|
||||
)
|
||||
record.data = {"name": "task", "id": "job", "return_value": URL}
|
||||
assert celery_worker.CeleryLogPrivacyFilter().filter(record)
|
||||
if log_level == logging.DEBUG:
|
||||
assert CANARY in record.getMessage()
|
||||
assert record.data["return_value"] == URL
|
||||
else:
|
||||
assert CANARY not in record.getMessage()
|
||||
assert record.data == {}
|
||||
|
||||
|
||||
def test_task_arguments_have_private_display_representations(monkeypatch):
|
||||
"""The actual task arguments are delivered but their event repr is hidden."""
|
||||
publish = Mock()
|
||||
monkeypatch.setattr(celery_worker.Task, "apply_async", publish)
|
||||
task = celery_worker.PrivateTask()
|
||||
task.apply_async(args=[CANARY], kwargs={"url": URL})
|
||||
assert publish.call_args.kwargs["args"] == [CANARY]
|
||||
assert publish.call_args.kwargs["argsrepr"] == "[Filtered]"
|
||||
assert publish.call_args.kwargs["kwargsrepr"] == "[Filtered]"
|
||||
|
||||
|
||||
def test_analytics_source_url_removes_signature():
|
||||
"""Task analytics keeps the source path without credentials or signature."""
|
||||
from datetime import datetime, timezone # noqa: PLC0415
|
||||
|
||||
from summary.core.models import TranscribeTaskJob # noqa: PLC0415
|
||||
|
||||
manager = analytics.MetadataManager()
|
||||
manager._is_disabled = False
|
||||
manager.has_task_id = Mock(return_value=False)
|
||||
manager._save_metadata = Mock()
|
||||
payload = TranscribeTaskJob(
|
||||
user_sub="user",
|
||||
tenant_id="tenant",
|
||||
cloud_storage_url=URL,
|
||||
language="en",
|
||||
received_at=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
manager.create("job", payload)
|
||||
metadata = manager._save_metadata.call_args.args[1]
|
||||
assert metadata["source_url"] == "https://example.com/file?[redacted]"
|
||||
assert CANARY not in str(metadata)
|
||||
|
||||
|
||||
def test_invalid_llm_json_is_sanitized():
|
||||
"""Invalid structured LLM output retains a retryable LLM category."""
|
||||
with pytest.raises(LLMException) as exc:
|
||||
celery_worker.parse_llm_json(CANARY)
|
||||
assert CANARY not in str(exc.value)
|
||||
assert exc.value.__suppress_context__
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
(None, "[redacted]"),
|
||||
("https://[invalid?secret=value", "[redacted]"),
|
||||
("https://example.com/path#fragment", "https://example.com/path"),
|
||||
("https://example.com/path?", "https://example.com/path?[redacted]"),
|
||||
],
|
||||
)
|
||||
def test_redact_url_edge_cases(url, expected):
|
||||
"""Malformed and non-string URLs do not expose credentials or query data."""
|
||||
assert redact_url(url) == expected
|
||||
@@ -1,32 +1,27 @@
|
||||
"""Tests for the Sentry configuration.
|
||||
|
||||
Each test raises an error from code handling meeting content and inspects the
|
||||
event Sentry would send: sensitive content is redacted and useful locals remain.
|
||||
event Sentry would send: the content must be redacted, while harmless local
|
||||
variables are kept for debugging.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Iterator
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from threading import Thread
|
||||
from unittest.mock import Mock
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import requests
|
||||
import sentry_sdk
|
||||
from botocore.exceptions import ClientError
|
||||
from botocore.stub import Stubber
|
||||
from sentry_sdk.integrations.logging import LoggingIntegration
|
||||
from sentry_sdk.integrations.stdlib import StdlibIntegration
|
||||
from sentry_sdk.transport import Transport
|
||||
|
||||
from summary.core import file_service, webhook_service
|
||||
from summary.core import file_service
|
||||
from summary.core import sentry as sentry_module
|
||||
from summary.core.file_service import FileService, FileServiceException
|
||||
from summary.core.file_service import FileService
|
||||
from summary.core.llm_service import LLMException, LLMService
|
||||
from summary.core.shared_models import SummarizeWebhookSuccessPayload, WhisperXResponse
|
||||
from summary.core.shared_models import WhisperXResponse
|
||||
|
||||
CANARY = "CANARY-MEETING-CONTENT"
|
||||
|
||||
@@ -56,14 +51,8 @@ def sentry_events() -> Iterator[SentryEvents]:
|
||||
transport=transport,
|
||||
send_default_pii=False,
|
||||
include_local_variables=True,
|
||||
before_send=sentry_module.before_send,
|
||||
before_breadcrumb=sentry_module.before_breadcrumb,
|
||||
event_scrubber=sentry_module.build_event_scrubber(),
|
||||
default_integrations=False,
|
||||
integrations=[
|
||||
LoggingIntegration(level=logging.DEBUG, event_level=None),
|
||||
StdlibIntegration(),
|
||||
],
|
||||
)
|
||||
|
||||
def flush() -> list[str]:
|
||||
@@ -74,125 +63,6 @@ def sentry_events() -> Iterator[SentryEvents]:
|
||||
sentry_sdk.init() # Disable Sentry for the following tests
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def missing_recording_url() -> Iterator[str]:
|
||||
"""Serve a deterministic 404 through the real instrumented HTTP client."""
|
||||
|
||||
class MissingRecordingHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
with ThreadingHTTPServer(("127.0.0.1", 0), MissingRecordingHandler) as server:
|
||||
thread = Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield (
|
||||
f"http://127.0.0.1:{server.server_port}/recordings/recording-123.mp4"
|
||||
f"?X-Amz-Credential={CANARY}-credential"
|
||||
f"&X-Amz-Signature={CANARY}-signature"
|
||||
)
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join()
|
||||
|
||||
|
||||
def test_sentry_download_404_excludes_signed_url(
|
||||
sentry_events: SentryEvents, missing_recording_url: str, caplog
|
||||
) -> None:
|
||||
"""A failed download excludes signed URLs from logs, locals and breadcrumbs."""
|
||||
with caplog.at_level(logging.INFO):
|
||||
try:
|
||||
FileService()._download_from_cloud_storage_url(missing_recording_url)
|
||||
except FileServiceException as exc:
|
||||
assert exc.__cause__ is None
|
||||
assert exc.__suppress_context__
|
||||
sentry_sdk.capture_exception()
|
||||
else:
|
||||
pytest.fail("the download should have failed")
|
||||
|
||||
[event] = sentry_events()
|
||||
assert CANARY not in event
|
||||
assert CANARY not in caplog.text
|
||||
assert "/recordings/recording-123.mp4?[redacted]" in caplog.text
|
||||
parsed = json.loads(event)
|
||||
assert [item["type"] for item in parsed["exception"]["values"]] == [
|
||||
"FileServiceException"
|
||||
]
|
||||
download_vars = next(
|
||||
frame_vars for frame_vars in _local_vars(event) if "extension" in frame_vars
|
||||
)
|
||||
assert download_vars["extension"] == "'.mp4'"
|
||||
assert download_vars["cloud_storage_url"] == "[Filtered]"
|
||||
breadcrumbs = parsed["breadcrumbs"]["values"]
|
||||
assert any(crumb.get("type") == "http" for crumb in breadcrumbs)
|
||||
assert any(
|
||||
crumb.get("category") == "summary.core.file_service" for crumb in breadcrumbs
|
||||
)
|
||||
assert all("message" not in crumb and "data" not in crumb for crumb in breadcrumbs)
|
||||
|
||||
|
||||
def test_sentry_chained_http_error_excludes_signed_url(
|
||||
sentry_events: SentryEvents, missing_recording_url: str
|
||||
) -> None:
|
||||
"""Sentry also filters signed URLs in explicitly chained HTTP errors."""
|
||||
try:
|
||||
try:
|
||||
response = requests.get(missing_recording_url, timeout=5)
|
||||
response.raise_for_status()
|
||||
except requests.HTTPError as exc:
|
||||
raise FileServiceException("Download failed") from exc
|
||||
except FileServiceException:
|
||||
sentry_sdk.capture_exception()
|
||||
|
||||
[event] = sentry_events()
|
||||
assert CANARY not in event
|
||||
exceptions = json.loads(event)["exception"]["values"]
|
||||
assert [item["type"] for item in exceptions] == [
|
||||
"HTTPError",
|
||||
"FileServiceException",
|
||||
]
|
||||
assert all(item["value"] == "[Filtered]" for item in exceptions)
|
||||
|
||||
|
||||
def test_sentry_excludes_webhook_debug_content(
|
||||
sentry_events: SentryEvents, monkeypatch, settings_override, caplog
|
||||
) -> None:
|
||||
"""Raw DEBUG responses stay in application logs and out of Sentry events."""
|
||||
monkeypatch.setattr(webhook_service, "settings", settings_override)
|
||||
signed_url = f"https://storage.example.com/result?signature={CANARY}"
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {
|
||||
"id": "document-123",
|
||||
"content": CANARY,
|
||||
"url": signed_url,
|
||||
}
|
||||
monkeypatch.setattr(webhook_service.requests, "post", Mock(return_value=response))
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
webhook_service.call_webhook_v2(
|
||||
tenant_id="test-tenant",
|
||||
payload=SummarizeWebhookSuccessPayload(
|
||||
job_id="job", summary_data_url=signed_url
|
||||
),
|
||||
)
|
||||
try:
|
||||
raise RuntimeError("A later failure")
|
||||
except RuntimeError:
|
||||
sentry_sdk.capture_exception()
|
||||
|
||||
assert CANARY in caplog.text
|
||||
[event] = sentry_events()
|
||||
assert CANARY not in event
|
||||
assert any(
|
||||
crumb.get("category") == "summary.core.webhook_service"
|
||||
and crumb.get("level") == "debug"
|
||||
for crumb in json.loads(event)["breadcrumbs"]["values"]
|
||||
)
|
||||
|
||||
|
||||
def _transcript() -> WhisperXResponse:
|
||||
return WhisperXResponse.model_validate(
|
||||
{
|
||||
@@ -251,7 +121,7 @@ def s3_stubber(monkeypatch: pytest.MonkeyPatch) -> Iterator[Stubber]:
|
||||
def test_sentry_store_transcript_failure_redacts_transcript(
|
||||
sentry_events: SentryEvents, s3_stubber: Stubber
|
||||
) -> None:
|
||||
"""A failed S3 upload redacts the transcript while retaining useful locals."""
|
||||
"""A failed S3 upload does not send the transcript, but keeps the job id."""
|
||||
s3_stubber.add_client_error("put_object", service_error_code="InvalidDigest")
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
"""Formatted validation errors hide input values, including after pickling."""
|
||||
|
||||
import pickle
|
||||
import traceback
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from summary.core.exceptions import CleanedValidationError
|
||||
|
||||
CANARY = "private-validation-canary"
|
||||
|
||||
|
||||
def test_formatted_errors_hide_ordinary_input_after_pickle_roundtrip():
|
||||
"""Formatting hides raw input even when diagnostics intentionally retain it."""
|
||||
original = ValidationError.from_exception_data(
|
||||
"TestModel",
|
||||
[{"type": "int_parsing", "loc": ("field",), "input": CANARY}],
|
||||
)
|
||||
cleaned = CleanedValidationError(original)
|
||||
restored = pickle.loads(pickle.dumps(cleaned)) # noqa: S301 -- own test exception
|
||||
for error in (cleaned, restored):
|
||||
assert error.errors()[0]["input"] == CANARY
|
||||
assert CANARY not in str(error)
|
||||
assert CANARY not in repr(error)
|
||||
assert CANARY not in "".join(traceback.format_exception(error))
|
||||
assert error.errors()[0]["type"] == "int_parsing"
|
||||
@@ -1,223 +0,0 @@
|
||||
"""Validation cleanup preserves diagnostics without retaining SecretStr values."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import pickle
|
||||
import traceback
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import pytest
|
||||
from celery.exceptions import Retry
|
||||
from celery.utils.serialization import get_pickleable_exception
|
||||
from fastapi import Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from pydantic import SecretStr, ValidationError
|
||||
from pydantic_core import PydanticCustomError
|
||||
|
||||
from summary.core import celery_worker
|
||||
from summary.core.exceptions import CleanedValidationError, clean_validation_errors
|
||||
from summary.main import validation_error_handler
|
||||
|
||||
CANARY = "private-validation-canary"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom", [False, True])
|
||||
def test_cleaned_validation_error_preserves_details_and_pickles(custom):
|
||||
"""Builtin and custom validation errors retain their type, location and message."""
|
||||
error_type = (
|
||||
PydanticCustomError(
|
||||
"custom_validation", "Invalid value: {reason}", {"reason": "test"}
|
||||
)
|
||||
if custom
|
||||
else "int_type"
|
||||
)
|
||||
original = ValidationError.from_exception_data(
|
||||
"TestModel",
|
||||
[
|
||||
{"type": error_type, "loc": ("secret",), "input": SecretStr(CANARY)},
|
||||
{"type": "int_parsing", "loc": ("ordinary",), "input": "not-an-int"},
|
||||
],
|
||||
)
|
||||
cleaned = CleanedValidationError(original)
|
||||
original_details = original.errors()
|
||||
details = cleaned.errors()
|
||||
assert isinstance(cleaned, ValidationError)
|
||||
assert cleaned.title == original.title
|
||||
assert cleaned.error_count() == original.error_count()
|
||||
assert details[0] == {**original_details[0], "input": "[redacted]"}
|
||||
assert details[1] == original_details[1]
|
||||
assert isinstance(original.errors()[0]["input"], SecretStr)
|
||||
assert CANARY not in str(cleaned)
|
||||
assert CANARY not in cleaned.json()
|
||||
serialized = pickle.dumps(cleaned)
|
||||
assert CANARY.encode() not in serialized
|
||||
restored = pickle.loads(serialized) # noqa: S301 -- own in-memory test exception
|
||||
assert isinstance(restored, CleanedValidationError)
|
||||
assert restored.errors() == details
|
||||
assert isinstance(get_pickleable_exception(cleaned), CleanedValidationError)
|
||||
|
||||
|
||||
def test_cleanup_masks_nested_secrets_and_preserves_plain_input():
|
||||
"""Missing-field inputs keep ordinary values while masking nested secrets."""
|
||||
original = ValidationError.from_exception_data(
|
||||
"NestedModel",
|
||||
[
|
||||
{
|
||||
"type": "missing",
|
||||
"loc": ("required",),
|
||||
"input": {
|
||||
"ordinary": "keep me",
|
||||
"nested": [SecretStr(CANARY), {"url": SecretStr(CANARY)}],
|
||||
"tuple": (SecretStr(CANARY), 42),
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
cleaned = CleanedValidationError(original)
|
||||
assert cleaned.errors()[0]["input"] == {
|
||||
"ordinary": "keep me",
|
||||
"nested": ["[redacted]", {"url": "[redacted]"}],
|
||||
"tuple": ("[redacted]", 42),
|
||||
}
|
||||
assert CANARY not in cleaned.json()
|
||||
assert CANARY.encode() not in pickle.dumps(cleaned)
|
||||
|
||||
|
||||
def test_cleanup_does_not_retain_validator_exception_objects():
|
||||
"""Validator context keeps its readable message without secret-bearing args."""
|
||||
original = ValidationError.from_exception_data(
|
||||
"TestModel",
|
||||
[
|
||||
{
|
||||
"type": "value_error",
|
||||
"loc": (),
|
||||
"input": SecretStr(CANARY),
|
||||
"ctx": {"error": ValueError(SecretStr(CANARY))},
|
||||
}
|
||||
],
|
||||
)
|
||||
cleaned = CleanedValidationError(original)
|
||||
assert cleaned.errors()[0]["ctx"] == {"error": "**********"}
|
||||
assert CANARY.encode() not in pickle.dumps(cleaned)
|
||||
|
||||
|
||||
def test_clean_validation_errors_keeps_all_other_fields():
|
||||
"""The API cleanup preserves diagnostics and nonsecret input."""
|
||||
details = [
|
||||
{
|
||||
"type": "test",
|
||||
"loc": ("body", "field"),
|
||||
"msg": "Invalid value",
|
||||
"url": "https://example.com/errors/test",
|
||||
"ctx": {"limit": 3},
|
||||
"input": "ordinary input",
|
||||
}
|
||||
]
|
||||
assert clean_validation_errors(details) == details
|
||||
assert (
|
||||
json.loads(json.dumps(clean_validation_errors(details)))[0]["input"]
|
||||
== "ordinary input"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"handler_name",
|
||||
[
|
||||
"call_webhook_v2_task",
|
||||
"process_audio_transcribe_v2_task",
|
||||
"summarize_v2_task",
|
||||
"task_started_transcript",
|
||||
"task_started_summary",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("secret_wrapped", [False, True])
|
||||
def test_task_validation_uses_cleaned_errors(handler_name, secret_wrapped):
|
||||
"""Task entry points and prerun signals consistently suppress the original error."""
|
||||
handler = getattr(celery_worker, handler_name)
|
||||
signed_url = f"https://storage.example.com/file?signature={CANARY}"
|
||||
value = SecretStr(signed_url) if secret_wrapped else signed_url
|
||||
payload = {"cloud_storage_url": value, "content": value}
|
||||
with pytest.raises(CleanedValidationError) as exc:
|
||||
if handler_name.startswith("task_started"):
|
||||
handler(task_id="job", args=[payload])
|
||||
elif handler_name == "call_webhook_v2_task":
|
||||
handler.run(
|
||||
payload={
|
||||
"type": "summary",
|
||||
"status": "success",
|
||||
"summary_data_url": value,
|
||||
},
|
||||
tenant_id="tenant",
|
||||
)
|
||||
elif handler_name == "summarize_v2_task":
|
||||
# Exercise validation without triggering the summary task's retries.
|
||||
handler._orig_run(payload=payload)
|
||||
else:
|
||||
handler.run(payload=payload)
|
||||
assert exc.value.__suppress_context__
|
||||
assert CANARY not in "".join(traceback.format_exception(exc.value))
|
||||
assert CANARY not in exc.value.json()
|
||||
assert CANARY.encode() not in pickle.dumps(exc.value)
|
||||
|
||||
|
||||
def test_upstream_transcription_validation_hides_raw_input(monkeypatch):
|
||||
"""Invalid ASR responses do not remain in structured validation errors."""
|
||||
storage = MagicMock()
|
||||
storage.prepare_audio_file.return_value.__enter__.return_value = (
|
||||
Mock(),
|
||||
{"duration": 1},
|
||||
)
|
||||
monkeypatch.setattr(celery_worker, "file_service", storage)
|
||||
monkeypatch.setattr(celery_worker, "metadata_manager", Mock())
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"text": {"content": CANARY}}
|
||||
monkeypatch.setattr(celery_worker.requests, "post", Mock(return_value=response))
|
||||
with pytest.raises(CleanedValidationError) as exc:
|
||||
celery_worker.transcribe_audio(
|
||||
task_id="job", language="en", cloud_storage_url="https://example.com/audio"
|
||||
)
|
||||
assert exc.value.errors()[0]["input"] == "[redacted]"
|
||||
assert CANARY not in exc.value.json()
|
||||
assert CANARY.encode() not in pickle.dumps(exc.value)
|
||||
|
||||
|
||||
def test_api_error_handler_uses_secret_aware_cleanup():
|
||||
"""API errors mask typed secrets while retaining other values and context."""
|
||||
errors = [
|
||||
{
|
||||
"type": "test",
|
||||
"loc": ("body", "field"),
|
||||
"msg": "Invalid value",
|
||||
"input": {"url": SecretStr(CANARY), "ordinary": "keep me"},
|
||||
"ctx": {"limit": 3},
|
||||
"url": "https://example.com/errors/test",
|
||||
},
|
||||
]
|
||||
response = asyncio.run(
|
||||
validation_error_handler(
|
||||
Request({"type": "http"}), RequestValidationError(errors)
|
||||
)
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert CANARY.encode() not in response.body
|
||||
assert json.loads(response.body)["detail"] == [
|
||||
{
|
||||
"type": "test",
|
||||
"loc": ["body", "field"],
|
||||
"msg": "Invalid value",
|
||||
"input": {"url": "[redacted]", "ordinary": "keep me"},
|
||||
"ctx": {"limit": 3},
|
||||
"url": "https://example.com/errors/test",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_summary_task_retries_with_the_cleaned_exception(monkeypatch):
|
||||
"""The summary task still retries validation failures using the safe exception."""
|
||||
retry = Mock(side_effect=Retry())
|
||||
monkeypatch.setattr(celery_worker.summarize_v2_task, "retry", retry)
|
||||
with pytest.raises(Retry):
|
||||
celery_worker.summarize_v2_task.run(payload={"received_at": SecretStr(CANARY)})
|
||||
error = retry.call_args.kwargs["exc"]
|
||||
assert isinstance(error, CleanedValidationError)
|
||||
assert CANARY.encode() not in pickle.dumps(error)
|
||||
Generated
+1
-1
@@ -1484,7 +1484,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "summary"
|
||||
version = "1.36.3"
|
||||
version = "1.37.2"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "boto3" },
|
||||
|
||||
Reference in New Issue
Block a user