From 72dd50e46589c7c07bbff264f33fa5e7316501fa Mon Sep 17 00:00:00 2001 From: Hanashi Date: Sun, 9 Aug 2026 21:04:14 -0400 Subject: [PATCH] test durations --- .../mysql_testing/src/fixtures/media_files.rs | 34 ++ .../common/probed_reference_videos.rs | 40 ++- .../image/omni_api_image_generate_handler.rs | 1 + .../video/omni_api_video_generate_handler.rs | 2 + .../cost/video/omni_gen_video_cost_handler.rs | 1 + .../audio/omni_gen_audio_generate_handler.rs | 1 + .../image/omni_gen_image_generate_handler.rs | 1 + .../mesh/omni_gen_mesh_generate_handler.rs | 1 + .../splat/omni_gen_splat_generate_handler.rs | 1 + .../video/omni_gen_video_generate_handler.rs | 2 + .../omni_gen/generate/video/tests/mod.rs | 4 +- .../tests/seedance_2p5/seedance_2p5_tests.rs | 124 ++++++- .../omni_gen/generate/video/tests/support.rs | 304 ++++++++++++++---- .../src/startup/build_dependencies.rs | 1 + .../storyteller_web/src/state/server_state.rs | 6 + ...kup_media_files_as_cdn_url_list_and_map.rs | 39 ++- 16 files changed, 483 insertions(+), 79 deletions(-) diff --git a/crates/schema/database/mysql_testing/src/fixtures/media_files.rs b/crates/schema/database/mysql_testing/src/fixtures/media_files.rs index 8da22818d8..bb61b15d75 100644 --- a/crates/schema/database/mysql_testing/src/fixtures/media_files.rs +++ b/crates/schema/database/mysql_testing/src/fixtures/media_files.rs @@ -15,6 +15,40 @@ use tokens::tokens::users::UserToken; use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; +/// Insert a video media_files row whose CDN path encodes `duration_millis` +/// as a `dur{millis}ms` prefix. Paired with a media-CDN override pointing at +/// a stub that serves real generated video of that duration, this lets tests +/// exercise the download + ffprobe billing path with exact durations. The +/// STORED duration column is deliberately wrong (999_000 ms) to prove the +/// database is never trusted for billing. +pub async fn create_probeable_test_video_media_file( + pool: &MySqlPool, + owner: &UserToken, + duration_millis: u64, +) -> anyhow::Result { + let prefix = format!("dur{duration_millis}ms"); + let bucket_path = MediaFileBucketPath::generate_new(Some(&prefix), Some("mp4")); + + MediaFileInsertBuilder::new() + .creator_user(owner) + .creator_ip_address("127.0.0.1") + .creator_set_visibility(Visibility::Private) + .media_file_class(MediaFileClass::Video) + .media_file_type(MediaFileType::Video) + .is_user_upload(true) + .is_intermediate_system_file(false) + .media_file_origin_category(MediaFileOriginCategory::Upload) + .media_file_origin_product_category(MediaFileOriginProductCategory::Unknown) + .mime_type("video/mp4") + .file_size_bytes(1_024) + .maybe_duration_millis(Some(999_000)) + .checksum_sha2(&"0".repeat(64)) + .public_bucket_directory_hash(&bucket_path) + .insert_pool(pool) + .await + .map_err(|err| anyhow!("media file insert failed: {err:?}")) +} + /// Insert a video media_files row owned by `owner`, with the given stored /// duration. Returns its token for use in `reference_video_media_tokens`. pub async fn create_test_video_media_file( diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/common/probed_reference_videos.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/common/probed_reference_videos.rs index 55e20dac4f..b10b33e6ec 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/common/probed_reference_videos.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/common/probed_reference_videos.rs @@ -20,8 +20,9 @@ //! once and ffprobes it. Callers should DROP their pooled connection //! before this phase and re-acquire after. //! -//! Billing math: each file's duration is rounded UP to a whole second, and -//! counted once per reference (the same file referenced twice bills twice). +//! Billing math: each file's duration is rounded UP to a whole second and +//! clamped to at least [`MIN_BILLED_INPUT_SECONDS_PER_VIDEO`], and counted +//! once per reference (the same file referenced twice bills twice). //! //! Probing NEVER fails a generation: a file whose download or ffprobe fails //! is billed at the worst-case [`MAX_BILLED_INPUT_SECONDS`] instead (and is @@ -44,6 +45,10 @@ use tempfile::NamedTempFile; use tokens::tokens::media_files::MediaFileToken; use crate::http_server::common_responses::common_web_error::CommonWebError; + +/// Every input video bills at least this many seconds, no matter how short +/// the file actually is. (Input durations run 1-30s; under 4s clamps to 4.) +pub const MIN_BILLED_INPUT_SECONDS_PER_VIDEO: u64 = 4; use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; use crate::util::http_download_url_to_tempfile::http_download_url_to_tempfile; @@ -87,6 +92,7 @@ pub async fn fetch_reference_video_sources( video_tokens: &[MediaFileToken], http_request: &HttpRequest, server_environment: ServerEnvironment, + maybe_media_cdn_override_url: Option<&str>, mysql_connection: &mut PoolConnection, ) -> Result, CommonWebError> { const CAN_SEE_DELETED: bool = false; @@ -133,7 +139,11 @@ pub async fn fetch_reference_video_sources( server_environment, &bucket_path); - (file.token, media_links.cdn_url.to_string()) + let cdn_url = crate::util::lookup::lookup_media_files_as_cdn_url_list_and_map::apply_media_cdn_override( + &media_links.cdn_url, + maybe_media_cdn_override_url, + ); + (file.token, cdn_url) }) .collect(); @@ -173,7 +183,9 @@ pub async fn download_and_probe_reference_videos( info!("Probed reference video {} at {}ms", source.media_token, duration_millis); local_paths_by_url.insert(source.cdn_url.clone(), temp_file.path().to_path_buf()); temp_files.push(temp_file); - duration_millis.div_ceil(1_000) + duration_millis + .div_ceil(1_000) + .max(MIN_BILLED_INPUT_SECONDS_PER_VIDEO) } Err(err) => { warn!( @@ -270,6 +282,22 @@ mod tests { assert_eq!(total_billed_seconds(&sources, &seconds), 20); } + #[test] + fn short_videos_clamp_to_the_per_video_minimum() { + // 1.5s and 3.9s each clamp to 4; a 4.1s file bills 5. + let sources = [ + source("m_a", "https://cdn/a.mp4"), + source("m_b", "https://cdn/b.mp4"), + source("m_c", "https://cdn/c.mp4"), + ]; + let seconds = seconds_map(&[ + ("https://cdn/a.mp4", 1_500), + ("https://cdn/b.mp4", 3_900), + ("https://cdn/c.mp4", 4_100), + ]); + assert_eq!(total_billed_seconds(&sources, &seconds), 4 + 4 + 5); + } + #[test] fn empty_sources_sum_to_zero() { assert_eq!(total_billed_seconds(&[], &HashMap::new()), 0); @@ -292,7 +320,9 @@ mod tests { fn seconds_map(entries: &[(&str, u64)]) -> HashMap { entries .iter() - .map(|(url, millis)| (url.to_string(), millis.div_ceil(1_000))) + .map(|(url, millis)| { + (url.to_string(), millis.div_ceil(1_000).max(MIN_BILLED_INPUT_SECONDS_PER_VIDEO)) + }) .collect() } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/generate/image/omni_api_image_generate_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/generate/image/omni_api_image_generate_handler.rs index 412f0c2df1..243ef8c449 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/generate/image/omni_api_image_generate_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/generate/image/omni_api_image_generate_handler.rs @@ -130,6 +130,7 @@ pub async fn omni_api_image_generate_handler( &http_request, &mut mysql_connection, server_state.server_environment, +server_state.maybe_media_cdn_override_url.as_deref(), request.image_media_tokens.as_deref().unwrap_or(&[]), ).await?; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/generate/video/omni_api_video_generate_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/generate/video/omni_api_video_generate_handler.rs index 46d809cf67..15f2716747 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/generate/video/omni_api_video_generate_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/generate/video/omni_api_video_generate_handler.rs @@ -171,6 +171,7 @@ pub async fn omni_api_video_generate_handler( &http_request, &mut mysql_connection, server_state.server_environment, +server_state.maybe_media_cdn_override_url.as_deref(), &all_tokens, ).await?; Some(resolved.token_to_url_map) @@ -199,6 +200,7 @@ pub async fn omni_api_video_generate_handler( video_tokens, &http_request, server_state.server_environment, + server_state.maybe_media_cdn_override_url.as_deref(), &mut mysql_connection, ).await?; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/cost/video/omni_gen_video_cost_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/cost/video/omni_gen_video_cost_handler.rs index 9aebd14ea4..d9c93c1671 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/cost/video/omni_gen_video_cost_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/cost/video/omni_gen_video_cost_handler.rs @@ -111,6 +111,7 @@ async fn probe_input_seconds_best_effort( video_tokens, http_request, server_state.server_environment, + server_state.maybe_media_cdn_override_url.as_deref(), &mut mysql_connection, ).await { Ok(sources) => sources, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/audio/omni_gen_audio_generate_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/audio/omni_gen_audio_generate_handler.rs index abc8744bd4..474875d2ac 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/audio/omni_gen_audio_generate_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/audio/omni_gen_audio_generate_handler.rs @@ -133,6 +133,7 @@ pub async fn omni_gen_audio_generate_handler( &http_request, &mut mysql_connection, server_state.server_environment, +server_state.maybe_media_cdn_override_url.as_deref(), &all_tokens, ).await?; Some(resolved.token_to_url_map) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/image/omni_gen_image_generate_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/image/omni_gen_image_generate_handler.rs index af783c0d56..e5b464f4fd 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/image/omni_gen_image_generate_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/image/omni_gen_image_generate_handler.rs @@ -121,6 +121,7 @@ pub async fn omni_gen_image_generate_handler( &http_request, &mut mysql_connection, server_state.server_environment, +server_state.maybe_media_cdn_override_url.as_deref(), request.image_media_tokens.as_deref().unwrap_or(&[]), ).await?; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/mesh/omni_gen_mesh_generate_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/mesh/omni_gen_mesh_generate_handler.rs index 72774bfeea..5c24801b7c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/mesh/omni_gen_mesh_generate_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/mesh/omni_gen_mesh_generate_handler.rs @@ -126,6 +126,7 @@ pub async fn omni_gen_mesh_generate_handler( &http_request, &mut mysql_connection, server_state.server_environment, +server_state.maybe_media_cdn_override_url.as_deref(), &all_tokens, ).await?; Some(resolved.token_to_url_map) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/omni_gen_splat_generate_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/omni_gen_splat_generate_handler.rs index 3134de038a..fd8d613b40 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/omni_gen_splat_generate_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/omni_gen_splat_generate_handler.rs @@ -131,6 +131,7 @@ pub async fn omni_gen_splat_generate_handler( &http_request, &mut mysql_connection, server_state.server_environment, +server_state.maybe_media_cdn_override_url.as_deref(), &all_tokens, ).await?; Some(resolved.token_to_url_map) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/omni_gen_video_generate_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/omni_gen_video_generate_handler.rs index 4ce9dc7957..00060d6e3f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/omni_gen_video_generate_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/omni_gen_video_generate_handler.rs @@ -154,6 +154,7 @@ pub async fn omni_gen_video_generate_handler( &http_request, &mut mysql_connection, server_state.server_environment, +server_state.maybe_media_cdn_override_url.as_deref(), &all_tokens, ).await?; Some(resolved.token_to_url_map) @@ -182,6 +183,7 @@ pub async fn omni_gen_video_generate_handler( video_tokens, &http_request, server_state.server_environment, + server_state.maybe_media_cdn_override_url.as_deref(), &mut mysql_connection, ).await?; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/mod.rs index b48441ed7e..b4a2693ad2 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/mod.rs @@ -13,7 +13,9 @@ //! //! Requirements: a local MySQL with an `artcraft_test` database reachable via //! `ARTCRAFT_TEST_DATABASE_URL` (default -//! `mysql://root:@localhost:3306/artcraft_test`). +//! `mysql://root:@localhost:3306/artcraft_test`), plus `ffmpeg`/`ffprobe` on +//! PATH (the stub CDN generates real fixture videos for the input-duration +//! billing tests, and the endpoint probes them). //! //! These tests run in PARALLEL: every test creates its own users, wallets, //! and media rows, schema setup is single-flighted via a MySQL named lock, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/seedance_2p5/seedance_2p5_tests.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/seedance_2p5/seedance_2p5_tests.rs index 3602ca12ad..a58d3eaf62 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/seedance_2p5/seedance_2p5_tests.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/seedance_2p5/seedance_2p5_tests.rs @@ -17,8 +17,9 @@ use enums::common::generation::common_resolution::CommonResolution; use enums::common::generation::common_video_model::CommonVideoModel; use crate::http_server::endpoints::omni_gen::generate::video::tests::support::{ - assert_reference_video_charge_then_refund, assert_successful_generation_charges, - assert_variant_charges_premium, Batch, CreditsDelta, ExpectedCredits, Seconds, TestHarness, + assert_real_input_videos_charge, assert_reference_video_charge_then_refund, + assert_successful_generation_charges, assert_variant_charges_premium, Batch, CreditsDelta, + ExpectedCredits, Seconds, TestHarness, }; // ── Seedance 2.5 ── @@ -929,6 +930,125 @@ mod seedance_2p5_u { } } + +// ── Input-video DURATION billing (real probed durations) ── +// The stub media CDN serves REAL generated videos with exact durations, so +// these run the full pipeline: download → ffprobe → billing → upload → +// generate. Billed input seconds per video = ceil(duration) clamped to a +// minimum of 4 (input durations run 1-30s; under 4s bills as 4s); the TOTAL +// across videos is capped at 30. Billed seconds = output + total input. +mod input_video_duration_billing { + use super::*; + + /// 1.5s and 3.9s inputs both bill the 4-second minimum: identical price. + #[tokio::test] + #[cfg_attr(feature = "skip_database_tests", ignore)] + async fn input_videos_under_four_seconds_clamp_to_four_seconds() { + let harness = TestHarness::create().await; + + // 30s output + 4s clamped input = 34 billed × 7.24279835 ¢/s → 247. + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::FourEightyP), + Seconds(30), &[1_500], ExpectedCredits(247), + ).await; + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::FourEightyP), + Seconds(30), &[3_900], ExpectedCredits(247), + ).await; + } + + /// Above the 4-second floor, each input bills its ceil-rounded duration. + #[tokio::test] + #[cfg_attr(feature = "skip_database_tests", ignore)] + async fn input_video_durations_bill_their_ceil_rounded_seconds() { + let harness = TestHarness::create().await; + + // (resolution, input millis, expected credits) at 30s output: + // 4.5s → 5 → 35 billed × 7.24279835 = 253.50 → 254 + // 6.5s → 7 → 37 billed × 7.24279835 = 267.98 → 268 + // 10.5s → 11 → 41 billed × 7.24279835 = 296.95 → 297 + // 6.5s → 7 → 37 billed × 15.84362140 = 586.21 → 587 (720p) + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::FourEightyP), + Seconds(30), &[4_500], ExpectedCredits(254), + ).await; + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::FourEightyP), + Seconds(30), &[6_500], ExpectedCredits(268), + ).await; + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::FourEightyP), + Seconds(30), &[10_500], ExpectedCredits(297), + ).await; + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::SevenTwentyP), + Seconds(30), &[6_500], ExpectedCredits(587), + ).await; + } + + /// The 4-second minimum applies PER VIDEO, before summing: a 2.5s + 6.5s + /// pair bills 4 + 7 = 11 input seconds — the same as a single 10.5s video, + /// and one more than a total-duration clamp (9s → 10) would produce. + #[tokio::test] + #[cfg_attr(feature = "skip_database_tests", ignore)] + async fn the_per_video_minimum_applies_before_summing() { + let harness = TestHarness::create().await; + + // 30s output + 11 input = 41 billed × 7.24279835 = 296.95 → 297. + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::FourEightyP), + Seconds(30), &[2_500, 6_500], ExpectedCredits(297), + ).await; + } + + /// Total input seconds cap at 30: a 29.5s video (→ 30) and a pair of 20s + /// videos (40 → capped 30) price identically to the worst case. + #[tokio::test] + #[cfg_attr(feature = "skip_database_tests", ignore)] + async fn total_input_seconds_cap_at_thirty() { + let harness = TestHarness::create().await; + + // 30s output + 30 input = 60 billed × 7.24279835 = 434.57 → 435. + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::FourEightyP), + Seconds(30), &[29_500], ExpectedCredits(435), + ).await; + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5, Some(CommonResolution::FourEightyP), + Seconds(30), &[20_000, 20_000], ExpectedCredits(435), + ).await; + } + + /// Ultra applies the same input-second rules at its own rates. + #[tokio::test] + #[cfg_attr(feature = "skip_database_tests", ignore)] + async fn ultra_bills_the_same_input_second_rules_at_its_rates() { + let harness = TestHarness::create().await; + + // 30s output, ultra with-refs 480p 8.55967078 ¢/s / 720p 18.72427984 ¢/s: + // 2.5s → 4 (clamp) → 34 billed → 291.03 → 292 + // 10.5s → 11 → 41 billed → 350.95 → 351 + // 2.5s + 6.5s → 11 → 41 billed → 351 (per-video clamp, same as 10.5s) + // 6.5s → 7 → 37 billed × 18.72427984 = 692.80 → 693 (720p) + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5Ultra, Some(CommonResolution::FourEightyP), + Seconds(30), &[2_500], ExpectedCredits(292), + ).await; + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5Ultra, Some(CommonResolution::FourEightyP), + Seconds(30), &[10_500], ExpectedCredits(351), + ).await; + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5Ultra, Some(CommonResolution::FourEightyP), + Seconds(30), &[2_500, 6_500], ExpectedCredits(351), + ).await; + assert_real_input_videos_charge( + &harness, CommonVideoModel::Seedance2p5Ultra, Some(CommonResolution::SevenTwentyP), + Seconds(30), &[6_500], ExpectedCredits(693), + ).await; + } +} + // ── Revenue premium: Ultra over the base model ── // Both prices and the delta are encoded so a change to EITHER rate card // shows up here, not just a flipped ordering. diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/support.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/support.rs index 53884a2c89..64ed091dc3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/support.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/support.rs @@ -88,7 +88,7 @@ impl TestHarness { let stub_kinovi_base_url = process_stub_kinovi_base_url().clone(); let pool = mysql_testing::pool::create_test_pool().await; - let server_state = Arc::new(build_test_server_state(pool.clone())); + let server_state = Arc::new(build_test_server_state(pool.clone(), stub_kinovi_base_url.clone())); TestHarness { pool, @@ -220,25 +220,33 @@ fn uuid_v4_like() -> String { format!("{:016x}{:016x}", nanos as u64, count) } -/// The process-wide stub Kinovi server's base URL. First use spawns the -/// stub (plain OS threads — it outlives every per-test tokio runtime) and -/// sets the Kinovi host-override env vars exactly once, before returning. +/// The process-wide stub server's base URL. First use spawns the stub +/// (plain OS threads — it outlives every per-test tokio runtime) and sets +/// BOTH the Kinovi host-override and media-CDN-override env vars exactly +/// once, before returning. fn process_stub_kinovi_base_url() -> &'static String { use std::sync::OnceLock; static STUB: OnceLock = OnceLock::new(); STUB.get_or_init(|| { - let base_url = spawn_stub_kinovi_server(); + let base_url = spawn_stub_server(); std::env::set_var(ENV_KINOVI_CUSTOM_API_HOST, &base_url); std::env::set_var(ENV_KINOVI_CUSTOM_CDN_HOST, &base_url); base_url }) } -/// A stub Kinovi API: accepts any POST and answers with a successful -/// workflow.runTask tRPC batch response with process-unique order ids. -/// Runs on plain OS threads so it is independent of any tokio runtime and -/// serves connections until the test process exits. -fn spawn_stub_kinovi_server() -> String { +/// A stub for BOTH the Kinovi API and the media CDN. Runs on plain OS +/// threads so it is independent of any tokio runtime and serves connections +/// until the test process exits. +/// +/// - `GET` paths containing `dur{millis}ms` serve a REAL generated video of +/// that duration (ffmpeg, cached per duration) — the reference-video +/// billing probe downloads and ffprobes these. +/// - `POST …uploads.signedUploadUrl` answers with an upload URL back at this +/// stub; the subsequent `PUT` is accepted with 200. +/// - Any other `POST` answers as a successful workflow.runTask with +/// process-unique order ids. +fn spawn_stub_server() -> String { use std::io::{Read, Write}; use std::net::TcpListener; use std::sync::atomic::{AtomicU64, Ordering}; @@ -252,17 +260,18 @@ fn spawn_stub_kinovi_server() -> String { .expect("clock") .as_nanos() as u64; - let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub kinovi"); + let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub server"); let port = listener.local_addr().expect("local addr").port(); + let base_url = format!("http://127.0.0.1:{port}"); + let base_url_for_thread = base_url.clone(); std::thread::spawn(move || { for stream in listener.incoming() { let Ok(mut stream) = stream else { continue }; + let base_url = base_url_for_thread.clone(); std::thread::spawn(move || { - // Read the request until the connection settles; every request gets - // the same shape of answer, so parsing is unnecessary. - let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); - let mut buffer = [0u8; 65536]; + let _ = stream.set_read_timeout(Some(Duration::from_millis(500))); + let mut buffer = vec![0u8; 1 << 20]; let mut read_total = 0; loop { match stream.read(&mut buffer[read_total..]) { @@ -275,30 +284,100 @@ fn spawn_stub_kinovi_server() -> String { _ => break, } } + let head = String::from_utf8_lossy(&buffer[..read_total]).to_string(); + let request_line = head.lines().next().unwrap_or_default().to_string(); + + let (status, content_type, body): (&str, &str, Vec) = + if request_line.starts_with("GET") { + match extract_fixture_duration_millis(&request_line) { + Some(millis) => ("200 OK", "video/mp4", fixture_video_bytes(millis)), + None => ("404 Not Found", "text/plain", b"not found".to_vec()), + } + } else if request_line.starts_with("PUT") { + ("200 OK", "text/plain", Vec::new()) + } else if request_line.contains("uploads.signedUploadUrl") { + let upload_url = format!("{base_url}/upload-target/materials/test/fixture.mp4"); + let json = format!(r#"[{{"result":{{"data":{{"json":"{upload_url}"}}}}}}]"#); + ("200 OK", "application/json", json.into_bytes()) + } else { + let order_number = ORDER_COUNTER.fetch_add(1, Ordering::Relaxed); + let json = format!( + r#"[{{"result":{{"data":{{"json":{{"taskId":"task_test_{process_prefix:016x}_{order_number:04}","orderId":"ord_test_{process_prefix:016x}_{order_number:04}","violationWarning":false}}}}}}}}]"#, + ); + ("200 OK", "application/json", json.into_bytes()) + }; - let order_number = ORDER_COUNTER.fetch_add(1, Ordering::Relaxed); - let body = format!( - r#"[{{"result":{{"data":{{"json":{{"taskId":"task_test_{process_prefix:016x}_{order_number:04}","orderId":"ord_test_{process_prefix:016x}_{order_number:04}","violationWarning":false}}}}}}}}]"#, - ); let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len(), - body, ); let _ = stream.write_all(response.as_bytes()); + let _ = stream.write_all(&body); let _ = stream.shutdown(std::net::Shutdown::Both); }); } }); - format!("http://127.0.0.1:{port}") + base_url +} + +/// `GET /media/.../dur2500ms_abc.mp4 HTTP/1.1` → `Some(2500)`. +fn extract_fixture_duration_millis(request_line: &str) -> Option { + let idx = request_line.find("dur")?; + let rest = &request_line[idx + 3..]; + let end = rest.find("ms")?; + rest[..end].parse().ok() +} + +/// A real MP4 of the requested duration, generated with ffmpeg once per +/// duration and cached on disk for the life of the machine's temp dir. +fn fixture_video_bytes(duration_millis: u64) -> Vec { + use std::sync::Mutex; + // Parallel tests race on cache generation; serialize it and land finished + // files with an atomic rename so a reader can never see a partial write. + static GENERATION_LOCK: Mutex<()> = Mutex::new(()); + + let cache_dir = std::env::temp_dir().join("artcraft_mysql_testing_fixture_videos"); + std::fs::create_dir_all(&cache_dir).expect("create fixture video cache dir"); + let cache_path = cache_dir.join(format!("fixture_{duration_millis}.mp4")); + + if !cache_path.exists() { + let _guard = GENERATION_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if !cache_path.exists() { + let scratch_path = cache_dir.join(format!( + "fixture_{duration_millis}.partial.{}.mp4", + std::process::id(), + )); + let seconds = duration_millis as f64 / 1_000.0; + let status = std::process::Command::new("ffmpeg") + .args([ + "-y", + "-f", "lavfi", + // Rate INSIDE the filter so the frame count (and thus the container + // duration) is exact; output retiming via `-r` pads the duration. + "-i", &format!("color=c=black:s=64x64:r=10:d={seconds}"), + "-pix_fmt", "yuv420p", + ]) + .arg(&scratch_path) + .output() + .expect("ffmpeg must be installed to generate fixture videos for database tests"); + assert!( + status.status.success(), + "ffmpeg failed generating a {duration_millis}ms fixture video: {}", + String::from_utf8_lossy(&status.stderr), + ); + std::fs::rename(&scratch_path, &cache_path).expect("land fixture video"); + } + } + + std::fs::read(&cache_path).expect("read fixture video") } /// A ServerState whose only live dependencies are the test MySQL pool and /// the stub Kinovi server. Everything else is inert dummy configuration: /// Redis/Elasticsearch/Stripe/bucket clients are constructed lazily and are /// never called on the video generate path. -fn build_test_server_state(pool: MySqlPool) -> ServerState { +fn build_test_server_state(pool: MySqlPool, media_cdn_override_url: String) -> ServerState { let session_cookie_manager = HttpUserSessionManager::new(TEST_COOKIE_DOMAIN, TEST_COOKIE_SECRET) .expect("session cookie manager"); let avt_cookie_manager = @@ -320,18 +399,71 @@ fn build_test_server_state(pool: MySqlPool) -> ServerState { let server_environment = ServerEnvironment::Development; let (pager, _pager_worker, paging_flags) = build_pager(server_environment, "test-host"); - let dummy_bucket = || { - LegacyBucketClient::create( - "test-access-key", - "test-secret-key", - "auto", - "test-bucket", - "http://127.0.0.1:1", - None, - Some(Duration::from_secs(5)), - ) - .expect("bucket client") - }; + // Built once per process: these construct HTTP/TLS clients that load the + // platform certificate store, which intermittently fails (macOS keychain + // I/O errors) when dozens of parallel tests do it simultaneously. + struct ExpensiveDummies { + stripe: StripeSettings, + stripe_artcraft: billing_artcraft_component::utils::artcraft_stripe_config::ArtcraftStripeConfigWithClient, + elasticsearch: Elasticsearch, + private_bucket_client: LegacyBucketClient, + public_bucket_client: LegacyBucketClient, + auto_gc_bucket_client: LegacyBucketClient, + redis_rate_limiters: crate::state::server_state::RedisRateLimiters, + } + use std::sync::OnceLock; + static DUMMIES: OnceLock = OnceLock::new(); + let dummies = DUMMIES.get_or_init(|| { + let dummy_bucket = || { + LegacyBucketClient::create( + "test-access-key", + "test-secret-key", + "auto", + "test-bucket", + "http://127.0.0.1:1", + None, + Some(Duration::from_secs(5)), + ) + .expect("bucket client") + }; + ExpensiveDummies { + stripe: StripeSettings { + config: StripeConfig { + checkout: StripeCheckoutConfigs { + success_url: FullUrlOrPath::Path("/success".to_string()), + cancel_url: FullUrlOrPath::Path("/cancel".to_string()), + }, + portal: StripeCustomerPortalConfigs { + return_url: FullUrlOrPath::Path("/portal".to_string()), + default_portal_config_id: "bpc_test".to_string(), + }, + secrets: StripeSecrets { + publishable_key: None, + secret_key: "sk_test_dummy".to_string(), + secret_webhook_signing_key: "whsec_test_dummy".to_string(), + }, + }, + client: stripe::Client::new("sk_test_dummy"), + stripe_account_id: "acct_test".to_string(), + }, + stripe_artcraft: ArtcraftStripeConfig { + stripe_account_id: "acct_test".to_string(), + secret_key: "sk_test_dummy".to_string(), + secret_webhook_signing_key: "whsec_test_dummy".to_string(), + checkout_success_url: "http://localhost/success".to_string(), + checkout_cancel_url: "http://localhost/cancel".to_string(), + portal_return_url: "http://localhost/portal".to_string(), + } + .to_config_with_client(), + elasticsearch: Elasticsearch::new( + Transport::single_node("http://127.0.0.1:1").expect("es transport"), + ), + private_bucket_client: dummy_bucket(), + public_bucket_client: dummy_bucket(), + auto_gc_bucket_client: dummy_bucket(), + redis_rate_limiters: configure_redis_rate_limiters().expect("rate limiters"), + } + }); ServerState { env_config: EnvConfig { @@ -343,55 +475,30 @@ fn build_test_server_state(pool: MySqlPool) -> ServerState { website_homepage_redirect: "http://localhost/".to_string(), }, server_info: ServerInfo { build_sha: "test".to_string() }, - stripe: StripeSettings { - config: StripeConfig { - checkout: StripeCheckoutConfigs { - success_url: FullUrlOrPath::Path("/success".to_string()), - cancel_url: FullUrlOrPath::Path("/cancel".to_string()), - }, - portal: StripeCustomerPortalConfigs { - return_url: FullUrlOrPath::Path("/portal".to_string()), - default_portal_config_id: "bpc_test".to_string(), - }, - secrets: StripeSecrets { - publishable_key: None, - secret_key: "sk_test_dummy".to_string(), - secret_webhook_signing_key: "whsec_test_dummy".to_string(), - }, - }, - client: stripe::Client::new("sk_test_dummy"), - stripe_account_id: "acct_test".to_string(), - }, - stripe_artcraft: ArtcraftStripeConfig { - stripe_account_id: "acct_test".to_string(), - secret_key: "sk_test_dummy".to_string(), - secret_webhook_signing_key: "whsec_test_dummy".to_string(), - checkout_success_url: "http://localhost/success".to_string(), - checkout_cancel_url: "http://localhost/cancel".to_string(), - portal_return_url: "http://localhost/portal".to_string(), - } - .to_config_with_client(), + stripe: dummies.stripe.clone(), + stripe_artcraft: dummies.stripe_artcraft.clone(), hostname: "test-host".to_string(), startup_time: Utc::now(), server_environment, + // Server-side media downloads (reference probing, upload resolution) hit + // the stub instead of the real CDN. + maybe_media_cdn_override_url: Some(media_cdn_override_url), flags: setup_static_feature_flags(paging_flags).expect("feature flags"), third_party_url_redirector: ThirdPartyUrlRedirector::new(server_environment), health_check_status: HealthCheckStatus::new(), mysql_pool: pool, - elasticsearch: Elasticsearch::new( - Transport::single_node("http://127.0.0.1:1").expect("es transport"), - ), + elasticsearch: dummies.elasticsearch.clone(), redis_pool, redis_ttl_cache, - redis_rate_limiters: configure_redis_rate_limiters().expect("rate limiters"), + redis_rate_limiters: dummies.redis_rate_limiters.clone(), session_cookie_manager, avt_cookie_manager, session_checker, firehose_publisher, badge_granter, - private_bucket_client: dummy_bucket(), - public_bucket_client: dummy_bucket(), - auto_gc_bucket_client: dummy_bucket(), + private_bucket_client: dummies.private_bucket_client.clone(), + public_bucket_client: dummies.public_bucket_client.clone(), + auto_gc_bucket_client: dummies.auto_gc_bucket_client.clone(), seedance_video_bucket: None, inference_providers: InferenceProviders { fal: FalData { @@ -611,3 +718,60 @@ pub async fn assert_variant_charges_premium( assert_successful_generation_charges(harness, base_model, resolution, seconds, batch, base).await; assert_successful_generation_charges(harness, variant_model, resolution, seconds, batch, variant).await; } + +/// Run one generation with REAL probeable reference videos (served by the +/// stub CDN with exact durations) and assert the exact wallet debit. The +/// whole flow succeeds: probe → billing → upload (to the stub) → generate. +pub async fn assert_real_input_videos_charge( + harness: &TestHarness, + model: CommonVideoModel, + resolution: Option, + Seconds(duration_seconds): Seconds, + input_video_millis: &[u64], + ExpectedCredits(expected_credits): ExpectedCredits, +) { + let user = harness.create_funded_user(STARTING_CREDITS).await; + + let mut video_tokens = Vec::new(); + for millis in input_video_millis { + let token = mysql_testing::fixtures::media_files::create_probeable_test_video_media_file( + &harness.pool, + &user.user_token, + *millis, + ) + .await + .expect("create probeable video media file fixture"); + video_tokens.push(token); + } + + let mut request = base_generate_request(model); + request.resolution = resolution; + request.duration_seconds = Some(duration_seconds); + request.reference_video_media_tokens = Some(video_tokens); + + let response = harness + .post_generate(&user, request) + .await + .unwrap_or_else(|err| { + panic!( + "{:?} {:?} {}s inputs {:?}ms: generation failed: {:?}", + model, resolution, duration_seconds, input_video_millis, err, + ) + }); + assert!(response.success); + + let balance = harness.wallet_balance(&user).await; + assert_eq!( + STARTING_CREDITS - balance, + expected_credits, + "{:?} {:?} {}s inputs {:?}ms: wrong wallet debit", + model, resolution, duration_seconds, input_video_millis, + ); + + let entries = harness.ledger_entries(&user).await; + let debit = entries + .iter() + .find(|entry| entry.credits_delta < 0) + .unwrap_or_else(|| panic!("{:?}: no debit ledger entry found", model)); + assert!(!debit.is_refunded, "{:?}: successful generation must not be refunded", model); +} diff --git a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs index bdcf11b4b4..b98a430832 100644 --- a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs +++ b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs @@ -215,6 +215,7 @@ pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult, + /// Feature flags will allow us to restart the service with different conditions embedded in the code. pub flags: StaticFeatureFlags, diff --git a/crates/service/web/storyteller_web/src/util/lookup/lookup_media_files_as_cdn_url_list_and_map.rs b/crates/service/web/storyteller_web/src/util/lookup/lookup_media_files_as_cdn_url_list_and_map.rs index a01e70defa..55a16f4752 100644 --- a/crates/service/web/storyteller_web/src/util/lookup/lookup_media_files_as_cdn_url_list_and_map.rs +++ b/crates/service/web/storyteller_web/src/util/lookup/lookup_media_files_as_cdn_url_list_and_map.rs @@ -27,6 +27,7 @@ pub async fn lookup_media_files_as_cdn_url_list_and_map( http_request: &HttpRequest, mysql_connection: &mut PoolConnection, server_environment: ServerEnvironment, + maybe_media_cdn_override_url: Option<&str>, tokens: &[MediaFileToken], ) -> Result { const CAN_SEE_DELETED: bool = false; @@ -92,7 +93,10 @@ pub async fn lookup_media_files_as_cdn_url_list_and_map( server_environment, &public_bucket_path); - token_to_url_map.insert(file.token, media_links.cdn_url.to_string()); + token_to_url_map.insert( + file.token, + apply_media_cdn_override(&media_links.cdn_url, maybe_media_cdn_override_url), + ); } // In requested order, with duplicates preserved. @@ -106,3 +110,36 @@ pub async fn lookup_media_files_as_cdn_url_list_and_map( token_to_url_map, }) } + +/// Rewrite a CDN URL onto the override base (keeping the path) when an +/// override is configured; otherwise pass the URL through unchanged. +pub fn apply_media_cdn_override(cdn_url: &url::Url, maybe_override_base: Option<&str>) -> String { + match maybe_override_base { + Some(base) => format!("{}{}", base.trim_end_matches('/'), cdn_url.path()), + None => cdn_url.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::apply_media_cdn_override; + + #[test] + fn no_override_passes_the_url_through() { + let url = url::Url::parse("https://cdn.example/media/a/b.mp4").unwrap(); + assert_eq!(apply_media_cdn_override(&url, None), "https://cdn.example/media/a/b.mp4"); + } + + #[test] + fn override_replaces_scheme_and_host_and_keeps_the_path() { + let url = url::Url::parse("https://cdn.example/media/a/b.mp4").unwrap(); + assert_eq!( + apply_media_cdn_override(&url, Some("http://127.0.0.1:5555")), + "http://127.0.0.1:5555/media/a/b.mp4", + ); + assert_eq!( + apply_media_cdn_override(&url, Some("http://127.0.0.1:5555/")), + "http://127.0.0.1:5555/media/a/b.mp4", + ); + } +}