diff --git a/.sqlx/query-31d85cc0013f29e6db2d272ee6e709cca79b46496a2c222960f1a1419175f00b.json b/.sqlx/query-31d85cc0013f29e6db2d272ee6e709cca79b46496a2c222960f1a1419175f00b.json new file mode 100644 index 0000000000..6d81f07e58 --- /dev/null +++ b/.sqlx/query-31d85cc0013f29e6db2d272ee6e709cca79b46496a2c222960f1a1419175f00b.json @@ -0,0 +1,135 @@ +{ + "db_name": "MySQL", + "query": "\nSELECT\n jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n\n jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`,\n jobs.maybe_external_third_party_id as `external_third_party_id`,\n\n jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n jobs.creator_ip_address,\n\n jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`,\n jobs.maybe_debug_log_event_token as `maybe_debug_log_event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM generic_inference_jobs as jobs\n\nWHERE jobs.maybe_external_third_party = ?\nAND jobs.maybe_external_third_party_id = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken", + "type_info": { + "type": "VarString", + "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status: enums::common::job_status_plus::JobStatusPlus", + "type_info": { + "type": "String", + "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | ENUM", + "char_set": 224, + "max_size": 76 + } + }, + { + "ordinal": 2, + "name": "external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty", + "type_info": { + "type": "VarString", + "flags": "MULTIPLE_KEY | BINARY", + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 3, + "name": "external_third_party_id", + "type_info": { + "type": "VarString", + "flags": "MULTIPLE_KEY | BINARY", + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_user_token: tokens::tokens::users::UserToken", + "type_info": { + "type": "VarString", + "flags": "MULTIPLE_KEY | BINARY", + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken", + "type_info": { + "type": "VarString", + "flags": "MULTIPLE_KEY | BINARY", + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 7, + "name": "maybe_prompt_token: tokens::tokens::prompts::PromptToken", + "type_info": { + "type": "VarString", + "flags": "BINARY", + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "maybe_debug_log_event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken", + "type_info": { + "type": "VarString", + "flags": "BINARY", + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP", + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 10, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW", + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false, + false, + true, + true, + true, + true, + false, + true, + true, + false, + false + ] + }, + "hash": "31d85cc0013f29e6db2d272ee6e709cca79b46496a2c222960f1a1419175f00b" +} diff --git a/Cargo.lock b/Cargo.lock index 8c83e1ec82..0e1fc85c2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1404,6 +1404,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "beeble_client" +version = "0.0.1" +dependencies = [ + "anyhow", + "errors", + "log", + "reqwest 0.12.23", + "serde", + "serde_derive", + "serde_json", + "test_data", + "tokio", + "workspace-build-acceleration", +] + [[package]] name = "bigdecimal" version = "0.3.1" @@ -11234,6 +11250,7 @@ dependencies = [ "async-stripe 0.26.0", "async-trait", "base64 0.22.1", + "beeble_client", "billing_artcraft_component", "billing_component", "bootstrap", diff --git a/Cargo.toml b/Cargo.toml index 2c5c28481c..19a56407ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/api_clients/grok_client", "crates/api_clients/midjourney_client", "crates/api_clients/muapi_client", + "crates/api_clients/beeble_client", "crates/api_clients/openai_sora_client", "crates/api_clients/openart_client", "crates/api_clients/rootly_client", @@ -244,6 +245,7 @@ fal_client = { path = "crates/api_clients/fal_client" } ffmpeg_utils = { path = "crates/lib/ffmpeg_utils" } file_to_base64_url = { path = "crates/lib/files/file_to_base64_url" } filesys = { path = "crates/lib/files/filesys" } +beeble_client = { path = "crates/api_clients/beeble_client" } grok_client = { path = "crates/api_clients/grok_client" } rootly_client = { path = "crates/api_clients/rootly_client" } rootly_config = { path = "crates/schema/config/rootly_config" } diff --git a/_tools/postman/collections/ArtCraft API/generate/video/edit/.resources/definition.yaml b/_tools/postman/collections/ArtCraft API/generate/video/edit/.resources/definition.yaml new file mode 100644 index 0000000000..974de54bef --- /dev/null +++ b/_tools/postman/collections/ArtCraft API/generate/video/edit/.resources/definition.yaml @@ -0,0 +1,3 @@ +$kind: collection +name: edit +order: 312 diff --git a/_tools/postman/collections/ArtCraft API/generate/video/edit/Beeble SwitchX Edit Video.request.yaml b/_tools/postman/collections/ArtCraft API/generate/video/edit/Beeble SwitchX Edit Video.request.yaml new file mode 100644 index 0000000000..6556711b2a --- /dev/null +++ b/_tools/postman/collections/ArtCraft API/generate/video/edit/Beeble SwitchX Edit Video.request.yaml @@ -0,0 +1,23 @@ +$kind: http-request +name: Beeble SwitchX Edit Video +url: "{{api_host}}/v1/generate/video/edit/beeble_switchx" +method: POST +headers: + - key: Accept + value: application/json + - key: Session + value: "" + disabled: true +body: + type: json + content: >- + { + "uuid_idempotency_token": "{{$randomUUID}}", + //"source_video_media_token": "m_6vve5gffn2r4rajyvy8z2380x15kqc", // two corgis (dev) + "source_video_media_token": "m_rkgdk7qw0a7gz4mrzfv0q78ax57kbx", // corgi cowboy (dev) + //"source_video_media_token": "m_1k19k6dgdny253zag8g3jqf3bgemss", // corgi watermellon (prod) + "reference_image_media_token": "m_pf310vdhc4v98n7r8mz70bz9geest3" // tornado (dev) + //"reference_image_media_token": "m_z4y097t9pdmbyatf70z0qb8zvfphvm" // cactus (prod) + //"prompt": "Stormy skies, there's a rotating tornado in the background" + } +order: 1778262498846 diff --git a/_tools/postman/collections/ArtCraft API/users/Login.request.yaml b/_tools/postman/collections/ArtCraft API/users/Login.request.yaml index 7a4fbcc6f9..7865c12e54 100644 --- a/_tools/postman/collections/ArtCraft API/users/Login.request.yaml +++ b/_tools/postman/collections/ArtCraft API/users/Login.request.yaml @@ -9,7 +9,7 @@ body: // hanashi : password (dev) //"username_or_email": "hanashi", //"password": "hanashi" - "username_or_email": "newuser2", - "password": "newuser2" + "username_or_email": "newuser4", + "password": "newuser4" } order: 1000 diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/edit/beeble_switchx_edit_video.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/edit/beeble_switchx_edit_video.rs new file mode 100644 index 0000000000..63c81fe119 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/edit/beeble_switchx_edit_video.rs @@ -0,0 +1,20 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const BEEBLE_SWITCHX_EDIT_VIDEO_PATH: &str = "/v1/generate/video/edit/beeble_switchx"; + +#[derive(Serialize, Deserialize, ToSchema, Debug)] +pub struct BeebleSwitchXEditVideoRequest { + pub uuid_idempotency_token: String, + pub source_video_media_token: Option, + pub reference_image_media_token: Option, + pub prompt: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct BeebleSwitchXEditVideoResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/edit/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/edit/mod.rs new file mode 100644 index 0000000000..ad8f1b29f4 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/edit/mod.rs @@ -0,0 +1 @@ +pub mod beeble_switchx_edit_video; diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/mod.rs index dcbdb40fbb..a1a46f0e8b 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/mod.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/mod.rs @@ -1,3 +1,4 @@ +pub mod edit; pub mod generate_kling_1_6_pro_image_to_video; pub mod generate_kling_2_1_master_image_to_video; pub mod generate_kling_2_1_pro_image_to_video; diff --git a/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/edit/beeble_switchx_edit_video.rs b/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/edit/beeble_switchx_edit_video.rs new file mode 100644 index 0000000000..d4ccaf0db8 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/edit/beeble_switchx_edit_video.rs @@ -0,0 +1,20 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::video::edit::beeble_switchx_edit_video::{ + BeebleSwitchXEditVideoRequest, BeebleSwitchXEditVideoResponse, BEEBLE_SWITCHX_EDIT_VIDEO_PATH, +}; + +pub async fn beeble_switchx_edit_video( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: BeebleSwitchXEditVideoRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + BEEBLE_SWITCHX_EDIT_VIDEO_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/edit/mod.rs b/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/edit/mod.rs new file mode 100644 index 0000000000..ad8f1b29f4 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/edit/mod.rs @@ -0,0 +1 @@ +pub mod beeble_switchx_edit_video; diff --git a/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/mod.rs b/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/mod.rs index 28a28bdbb2..ebae4b56f8 100644 --- a/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/mod.rs +++ b/crates/api_clients/artcraft/artcraft_client/src/endpoints/generate/video/mod.rs @@ -1,3 +1,4 @@ +pub mod edit; pub mod generate_kling_16_pro_image_to_video; pub mod generate_kling_21_master_image_to_video; pub mod generate_kling_21_pro_image_to_video; diff --git a/crates/api_clients/beeble_client/CLAUDE.md b/crates/api_clients/beeble_client/CLAUDE.md new file mode 100644 index 0000000000..cd77e5c204 --- /dev/null +++ b/crates/api_clients/beeble_client/CLAUDE.md @@ -0,0 +1,24 @@ +# beeble_client + +HTTP client for the Beeble SwitchX compositing API. + +## API Reference + +- Docs: https://developer.beeble.ai/docs +- OpenAPI: https://api.beeble.ai/developer-api-docs/openapi.json +- Auth: `x-api-key` header + +## Endpoints + +- `create_upload_url` — POST /v1/uploads — get a presigned PUT URL for uploading media +- `start_generation` — POST /v1/switchx/generations — start an image or video compositing job +- `get_job_status` — GET /v1/switchx/generations/{job_id} — poll job status until completed/failed + +## Testing + +```bash +cargo test -p beeble_client # run non-ignored tests (API shape only) +cargo test -p beeble_client -- --ignored # run live API tests (requires key, incurs costs) +``` + +Live tests require: `/Users/bt/Artcraft/credentials/beeble_api_key.txt` diff --git a/crates/api_clients/beeble_client/Cargo.toml b/crates/api_clients/beeble_client/Cargo.toml new file mode 100644 index 0000000000..5dfc201e09 --- /dev/null +++ b/crates/api_clients/beeble_client/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "beeble_client" +edition = "2024" +version = "0.0.1" +publish = false + +[package.metadata.cargo-udeps.ignore] +normal = ["workspace-build-acceleration"] + +[lib] +name = "beeble_client" +path = "src/lib.rs" + +[dependencies] +# Workspace hack (faster compile times) +workspace-build-acceleration.workspace = true + +# Internal +errors.workspace = true + +# External +log.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_derive.workspace = true +serde_json.workspace = true + +[dev-dependencies] +anyhow.workspace = true +test_data.workspace = true +tokio = { version = "1", features = ["full"] } diff --git a/crates/api_clients/beeble_client/src/creds/beeble_api_key.rs b/crates/api_clients/beeble_client/src/creds/beeble_api_key.rs new file mode 100644 index 0000000000..85cd82119a --- /dev/null +++ b/crates/api_clients/beeble_client/src/creds/beeble_api_key.rs @@ -0,0 +1,24 @@ +use std::fmt; + +/// Holds the API key for authenticating with the Beeble API. +/// Sent as the `x-api-key` header. +#[derive(Clone)] +pub struct BeebleApiKey { + pub(crate) api_key: String, +} + +impl BeebleApiKey { + pub fn new(api_key: String) -> Self { + Self { api_key } + } +} + +impl fmt::Debug for BeebleApiKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.api_key.len() > 0 { + write!(f, "BeebleApiKey(***)") + } else { + write!(f, "BeebleApiKey(EMPTY)") + } + } +} diff --git a/crates/api_clients/beeble_client/src/creds/mod.rs b/crates/api_clients/beeble_client/src/creds/mod.rs new file mode 100644 index 0000000000..a4d53a70b5 --- /dev/null +++ b/crates/api_clients/beeble_client/src/creds/mod.rs @@ -0,0 +1 @@ +pub mod beeble_api_key; diff --git a/crates/api_clients/beeble_client/src/error/beeble_client_error.rs b/crates/api_clients/beeble_client/src/error/beeble_client_error.rs new file mode 100644 index 0000000000..58fb17de62 --- /dev/null +++ b/crates/api_clients/beeble_client/src/error/beeble_client_error.rs @@ -0,0 +1,26 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; + +#[derive(Debug)] +pub enum BeebleClientError { + /// No API key is present. + NoApiKeyPresent, + + /// An error was encountered in building the reqwest client. + ReqwestClientError(reqwest::Error), + + /// A serialization error with the request. + SerializationError(serde_json::Error), +} + +impl Error for BeebleClientError {} + +impl Display for BeebleClientError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoApiKeyPresent => write!(f, "No API key present."), + Self::ReqwestClientError(err) => write!(f, "Reqwest client error: {}", err), + Self::SerializationError(err) => write!(f, "Serialization error: {}", err), + } + } +} diff --git a/crates/api_clients/beeble_client/src/error/beeble_error.rs b/crates/api_clients/beeble_client/src/error/beeble_error.rs new file mode 100644 index 0000000000..dddca7ab04 --- /dev/null +++ b/crates/api_clients/beeble_client/src/error/beeble_error.rs @@ -0,0 +1,41 @@ +use crate::error::beeble_client_error::BeebleClientError; +use crate::error::beeble_generic_api_error::BeebleGenericApiError; +use crate::error::beeble_specific_api_error::BeebleSpecificApiError; +use std::error::Error; + +#[derive(Debug)] +pub enum BeebleError { + Client(BeebleClientError), + ApiSpecific(BeebleSpecificApiError), + ApiGeneric(BeebleGenericApiError), +} + +impl Error for BeebleError {} + +impl std::fmt::Display for BeebleError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Client(e) => write!(f, "BeebleClientError: {:?}", e), + Self::ApiSpecific(e) => write!(f, "BeebleSpecificApiError: {:?}", e), + Self::ApiGeneric(e) => write!(f, "BeebleGenericApiError: {:?}", e), + } + } +} + +impl From for BeebleError { + fn from(error: BeebleClientError) -> Self { + Self::Client(error) + } +} + +impl From for BeebleError { + fn from(error: BeebleSpecificApiError) -> Self { + Self::ApiSpecific(error) + } +} + +impl From for BeebleError { + fn from(error: BeebleGenericApiError) -> Self { + Self::ApiGeneric(error) + } +} diff --git a/crates/api_clients/beeble_client/src/error/beeble_generic_api_error.rs b/crates/api_clients/beeble_client/src/error/beeble_generic_api_error.rs new file mode 100644 index 0000000000..7184ec916d --- /dev/null +++ b/crates/api_clients/beeble_client/src/error/beeble_generic_api_error.rs @@ -0,0 +1,34 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; + +/// Catch-all for API errors that don't fit a specific variant. +#[derive(Debug)] +pub enum BeebleGenericApiError { + /// An HTTP transport error from reqwest. + ReqwestError(reqwest::Error), + + /// Failed to parse the response body as JSON. + SerdeResponseParseError(serde_json::Error, String), + + /// The API returned a non-success status code. + UncategorizedBadResponseWithStatusAndBody { + status_code: reqwest::StatusCode, + body: String, + }, +} + +impl Error for BeebleGenericApiError {} + +impl Display for BeebleGenericApiError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::ReqwestError(err) => write!(f, "Beeble reqwest error: {}", err), + Self::SerdeResponseParseError(err, body) => { + write!(f, "Beeble response parse error: {} (body: {})", err, body) + } + Self::UncategorizedBadResponseWithStatusAndBody { status_code, body } => { + write!(f, "Beeble API error: status={}, body={}", status_code, body) + } + } + } +} diff --git a/crates/api_clients/beeble_client/src/error/beeble_specific_api_error.rs b/crates/api_clients/beeble_client/src/error/beeble_specific_api_error.rs new file mode 100644 index 0000000000..3c527ac13c --- /dev/null +++ b/crates/api_clients/beeble_client/src/error/beeble_specific_api_error.rs @@ -0,0 +1,54 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; + +/// Well-known API errors from the Beeble service. +#[derive(Debug)] +pub enum BeebleSpecificApiError { + /// 401 Unauthorized — invalid or missing API key. + Unauthorized, + + /// 402 Payment Required — insufficient credits. + InsufficientCredits, + + /// 409 Conflict — idempotency key reused with different parameters. + IdempotencyConflict, + + /// 429 Too Many Requests — rate limit exceeded. + RateLimited, + + /// The generation job failed with a reported error message. + GenerationFailed(String), + + /// 400 Bad Request — the callback_url is not a valid publicly-reachable HTTPS URL. + BadWebhookUrl { + message: String, + webhook_url: String, + }, + + /// 400 Bad Request — the video has too many frames. + VideoHasTooManyFrames { + max_frames: u64, + detected_frames: u64, + message: String, + }, +} + +impl Error for BeebleSpecificApiError {} + +impl Display for BeebleSpecificApiError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Unauthorized => write!(f, "Beeble API: Unauthorized (invalid or missing API key)"), + Self::InsufficientCredits => write!(f, "Beeble API: Insufficient credits"), + Self::IdempotencyConflict => write!(f, "Beeble API: Idempotency conflict"), + Self::RateLimited => write!(f, "Beeble API: Rate limited"), + Self::GenerationFailed(msg) => write!(f, "Beeble API: Generation failed: {}", msg), + Self::BadWebhookUrl { message, webhook_url } => { + write!(f, "Beeble API: Bad webhook URL '{}': {}", webhook_url, message) + } + Self::VideoHasTooManyFrames { max_frames, detected_frames, message } => { + write!(f, "Beeble API: Video has too many frames ({} detected, max {}): {}", detected_frames, max_frames, message) + } + } + } +} diff --git a/crates/api_clients/beeble_client/src/error/mod.rs b/crates/api_clients/beeble_client/src/error/mod.rs new file mode 100644 index 0000000000..6512780cfa --- /dev/null +++ b/crates/api_clients/beeble_client/src/error/mod.rs @@ -0,0 +1,4 @@ +pub mod beeble_client_error; +pub mod beeble_error; +pub mod beeble_generic_api_error; +pub mod beeble_specific_api_error; diff --git a/crates/api_clients/beeble_client/src/lib.rs b/crates/api_clients/beeble_client/src/lib.rs new file mode 100644 index 0000000000..5b3f02f548 --- /dev/null +++ b/crates/api_clients/beeble_client/src/lib.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +pub(crate) mod test_utils; + +// Public API +pub mod creds; +pub mod error; +pub mod requests; +pub mod webhook_api; diff --git a/crates/api_clients/beeble_client/src/requests/create_upload_url/create_upload_url.rs b/crates/api_clients/beeble_client/src/requests/create_upload_url/create_upload_url.rs new file mode 100644 index 0000000000..f4427aa4fe --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/create_upload_url/create_upload_url.rs @@ -0,0 +1,126 @@ +use log::info; + +use crate::creds::beeble_api_key::BeebleApiKey; +use crate::error::beeble_client_error::BeebleClientError; +use crate::error::beeble_error::BeebleError; +use crate::error::beeble_generic_api_error::BeebleGenericApiError; +use crate::error::beeble_specific_api_error::BeebleSpecificApiError; +use crate::requests::create_upload_url::request_types::*; + +const BEEBLE_API_BASE_URL: &str = "https://api.beeble.ai/v1"; + +// ── Public args ── + +pub struct CreateUploadUrlArgs { + pub api_key: BeebleApiKey, + pub filename: String, +} + +// ── Public response ── + +#[derive(Debug, Clone)] +pub struct CreateUploadUrlSuccess { + /// Upload ID (upload_...). + pub id: String, + /// Presigned PUT URL for uploading. Expires after 1 hour. + pub upload_url: String, + /// beeble:// URI to reference this upload in generation calls. + pub beeble_uri: String, +} + +// ── Implementation ── + +/// Create a presigned upload URL for a media file. +/// +/// After receiving the URL, PUT your file bytes directly to `upload_url`. +/// Then use `beeble_uri` as the source_uri, reference_image_uri, or alpha_uri +/// in generation calls. +pub async fn create_upload_url(args: CreateUploadUrlArgs) -> Result { + let url = format!("{}/uploads", BEEBLE_API_BASE_URL); + + info!("Creating Beeble upload URL for filename: {}", args.filename); + + let request_body = CreateUploadUrlRequestBody { + filename: args.filename, + }; + + let client = reqwest::Client::builder() + .build() + .map_err(|err| BeebleClientError::ReqwestClientError(err))?; + + let response = client.post(&url) + .header("x-api-key", &args.api_key.api_key) + .header("Content-Type", "application/json") + .json(&request_body) + .send() + .await + .map_err(|err| BeebleGenericApiError::ReqwestError(err))?; + + let status = response.status(); + let response_body = response.text() + .await + .map_err(|err| BeebleGenericApiError::ReqwestError(err))?; + + info!("Beeble create upload URL response: status={}", status); + + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(BeebleSpecificApiError::Unauthorized.into()); + } + + if !status.is_success() { + return Err(BeebleGenericApiError::UncategorizedBadResponseWithStatusAndBody { + status_code: status, + body: response_body, + }.into()); + } + + let parsed: CreateUploadUrlResponseBody = serde_json::from_str(&response_body) + .map_err(|err| BeebleGenericApiError::SerdeResponseParseError(err, response_body.clone()))?; + + Ok(CreateUploadUrlSuccess { + id: parsed.id, + upload_url: parsed.upload_url, + beeble_uri: parsed.beeble_uri, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::get_test_api_key::get_test_api_key; + use errors::AnyhowResult; + + #[test] + fn request_body_serializes() { + let body = CreateUploadUrlRequestBody { + filename: "test_video.mp4".to_string(), + }; + let json = serde_json::to_string(&body).unwrap(); + assert!(json.contains("test_video.mp4")); + } + + #[test] + fn response_body_deserializes() { + let json = r#"{"id":"upload_abc123","upload_url":"https://example.com/put","beeble_uri":"beeble://upload_abc123/video.mp4"}"#; + let parsed: CreateUploadUrlResponseBody = serde_json::from_str(json).unwrap(); + assert_eq!(parsed.id, "upload_abc123"); + assert_eq!(parsed.beeble_uri, "beeble://upload_abc123/video.mp4"); + } + + #[tokio::test] + #[ignore] // manually test — requires real API key + async fn test_create_upload_url() -> AnyhowResult<()> { + let api_key = get_test_api_key()?; + let result = create_upload_url(CreateUploadUrlArgs { + api_key, + filename: "test_image.png".to_string(), + }).await?; + + println!("Upload ID: {}", result.id); + println!("Upload URL: {}", result.upload_url); + println!("Beeble URI: {}", result.beeble_uri); + assert!(result.id.starts_with("upload_")); + assert!(result.beeble_uri.starts_with("beeble://")); + Ok(()) + } +} diff --git a/crates/api_clients/beeble_client/src/requests/create_upload_url/mod.rs b/crates/api_clients/beeble_client/src/requests/create_upload_url/mod.rs new file mode 100644 index 0000000000..f79471678d --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/create_upload_url/mod.rs @@ -0,0 +1,3 @@ +pub mod create_upload_url; +mod request_types; +pub mod upload_bytes; diff --git a/crates/api_clients/beeble_client/src/requests/create_upload_url/request_types.rs b/crates/api_clients/beeble_client/src/requests/create_upload_url/request_types.rs new file mode 100644 index 0000000000..709728997e --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/create_upload_url/request_types.rs @@ -0,0 +1,13 @@ +use serde_derive::{Deserialize, Serialize}; + +#[derive(Serialize)] +pub struct CreateUploadUrlRequestBody { + pub filename: String, +} + +#[derive(Deserialize, Debug)] +pub struct CreateUploadUrlResponseBody { + pub id: String, + pub upload_url: String, + pub beeble_uri: String, +} diff --git a/crates/api_clients/beeble_client/src/requests/create_upload_url/upload_bytes.rs b/crates/api_clients/beeble_client/src/requests/create_upload_url/upload_bytes.rs new file mode 100644 index 0000000000..921cb1b62c --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/create_upload_url/upload_bytes.rs @@ -0,0 +1,50 @@ +use log::info; + +use crate::creds::beeble_api_key::BeebleApiKey; +use crate::error::beeble_client_error::BeebleClientError; +use crate::error::beeble_error::BeebleError; +use crate::error::beeble_generic_api_error::BeebleGenericApiError; +use crate::requests::create_upload_url::create_upload_url::{ + create_upload_url, CreateUploadUrlArgs, CreateUploadUrlSuccess, +}; + +/// Upload bytes to Beeble in one call (create presigned URL + PUT bytes). +/// +/// Returns the upload success with the `beeble_uri` to use in generation calls. +pub async fn upload_bytes_to_beeble( + api_key: &BeebleApiKey, + filename: &str, + content_type: &str, + bytes: Vec, +) -> Result { + info!("Uploading {} bytes to Beeble as {}", bytes.len(), filename); + + // 1. Get presigned URL. + let upload = create_upload_url(CreateUploadUrlArgs { + api_key: api_key.clone(), + filename: filename.to_string(), + }).await?; + + info!("Beeble upload URL obtained: {}", upload.beeble_uri); + + // 2. PUT bytes to presigned URL. + let client = reqwest::Client::new(); + let put_response = client.put(&upload.upload_url) + .header("Content-Type", content_type) + .body(bytes) + .send() + .await + .map_err(|err| BeebleGenericApiError::ReqwestError(err))?; + + let status = put_response.status(); + if !status.is_success() { + let body = put_response.text().await.unwrap_or_default(); + return Err(BeebleGenericApiError::UncategorizedBadResponseWithStatusAndBody { + status_code: status, + body, + }.into()); + } + + info!("Beeble upload PUT succeeded: {}", upload.beeble_uri); + Ok(upload) +} diff --git a/crates/api_clients/beeble_client/src/requests/get_job_status/get_job_status.rs b/crates/api_clients/beeble_client/src/requests/get_job_status/get_job_status.rs new file mode 100644 index 0000000000..c72d59de79 --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/get_job_status/get_job_status.rs @@ -0,0 +1,104 @@ +use log::info; + +use crate::creds::beeble_api_key::BeebleApiKey; +use crate::error::beeble_client_error::BeebleClientError; +use crate::error::beeble_error::BeebleError; +use crate::error::beeble_generic_api_error::BeebleGenericApiError; +use crate::error::beeble_specific_api_error::BeebleSpecificApiError; +use crate::requests::start_generation::request_types::GenerationJobResponseBody; +use crate::requests::start_generation::start_generation::{map_job_response, StartGenerationSuccess}; + +const BEEBLE_API_BASE_URL: &str = "https://api.beeble.ai/v1"; + +// ── Public args ── + +pub struct GetJobStatusArgs { + pub api_key: BeebleApiKey, + /// The job ID (swx_...) to poll. + pub job_id: String, +} + +// ── Implementation ── + +/// Poll the status of a SwitchX generation job. +/// +/// Returns the same response shape as start_generation. +/// Poll until `status` is "completed" or "failed". +pub async fn get_job_status(args: GetJobStatusArgs) -> Result { + let url = format!("{}/switchx/generations/{}", BEEBLE_API_BASE_URL, args.job_id); + + info!("Polling Beeble job status: job_id={}", args.job_id); + + let client = reqwest::Client::builder() + .build() + .map_err(|err| BeebleClientError::ReqwestClientError(err))?; + + let response = client.get(&url) + .header("x-api-key", &args.api_key.api_key) + .send() + .await + .map_err(|err| BeebleGenericApiError::ReqwestError(err))?; + + let status = response.status(); + let response_body = response.text() + .await + .map_err(|err| BeebleGenericApiError::ReqwestError(err))?; + + info!("Beeble get job status response: status={}", status); + + match status.as_u16() { + 401 => return Err(BeebleSpecificApiError::Unauthorized.into()), + 429 => return Err(BeebleSpecificApiError::RateLimited.into()), + _ if !status.is_success() => { + return Err(BeebleGenericApiError::UncategorizedBadResponseWithStatusAndBody { + status_code: status, + body: response_body, + }.into()); + } + _ => {} + } + + let parsed: GenerationJobResponseBody = serde_json::from_str(&response_body) + .map_err(|err| BeebleGenericApiError::SerdeResponseParseError(err, response_body.clone()))?; + + Ok(map_job_response(parsed)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::get_test_api_key::get_test_api_key; + + //const TEST_JOB_ID: &str = "swx_uZAcwMjRZpDesqfp3pKkF"; // Production + //const TEST_JOB_ID: &str = "swx_Ontb6KfbfdY24Cz8DZ9c8"; // Dev + //const TEST_JOB_ID: &str = "swx_k5SmgdFcesuE05BE8VqFb"; // Dev + //const TEST_JOB_ID: &str = "swx_snkg7Et8EOvWaQ0QgoZlZ"; // Dev + const TEST_JOB_ID: &str = "swx_a1GBhEHwIfhzXfWEkEbcb"; // Dev + //const TEST_JOB_ID: &str = "swx_9Cbr7coiW9ZOayIrKGe0v"; // Prod + + #[tokio::test] + #[ignore] // manually test — requires real API key + async fn test_poll_known_job() -> errors::AnyhowResult<()> { + let api_key = get_test_api_key()?; + let result = get_job_status(GetJobStatusArgs { + api_key, + job_id: TEST_JOB_ID.to_string(), + }).await?; + + println!("Job ID: {}", result.id); + println!("Status: {}", result.status); + println!("Progress: {:?}", result.progress); + println!("Generation type: {:?}", result.generation_type); + println!("Alpha mode: {:?}", result.alpha_mode); + println!("Error: {:?}", result.error); + println!("Created at: {:?}", result.created_at); + if let Some(output) = &result.output { + println!("Render URL: {:?}", output.render); + println!("Source URL: {:?}", output.source); + println!("Alpha URL: {:?}", output.alpha); + } + + assert_eq!(result.id, TEST_JOB_ID); + Ok(()) + } +} diff --git a/crates/api_clients/beeble_client/src/requests/get_job_status/mod.rs b/crates/api_clients/beeble_client/src/requests/get_job_status/mod.rs new file mode 100644 index 0000000000..cb3bdb89ce --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/get_job_status/mod.rs @@ -0,0 +1 @@ +pub mod get_job_status; diff --git a/crates/api_clients/beeble_client/src/requests/mod.rs b/crates/api_clients/beeble_client/src/requests/mod.rs new file mode 100644 index 0000000000..3900ac031a --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/mod.rs @@ -0,0 +1,3 @@ +pub mod create_upload_url; +pub mod get_job_status; +pub mod start_generation; diff --git a/crates/api_clients/beeble_client/src/requests/start_generation/handle_error_response.rs b/crates/api_clients/beeble_client/src/requests/start_generation/handle_error_response.rs new file mode 100644 index 0000000000..efaffaacc6 --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/start_generation/handle_error_response.rs @@ -0,0 +1,72 @@ +use reqwest::StatusCode; + +use crate::error::beeble_error::BeebleError; +use crate::error::beeble_generic_api_error::BeebleGenericApiError; +use crate::error::beeble_specific_api_error::BeebleSpecificApiError; + +/// Additional context for error handling that depends on the original request. +pub struct ErrorContext <'a> { + pub maybe_callback_url: Option<&'a str>, +} + +/// Classify a non-success Beeble API response into the appropriate error. +/// +/// Only call this when `!status.is_success()`. Always returns `Err`. +pub fn handle_error_response( + status: StatusCode, + response_body: &str, + context: &ErrorContext, +) -> Result { + match status.as_u16() { + 401 => Err(BeebleSpecificApiError::Unauthorized.into()), + 402 => Err(BeebleSpecificApiError::InsufficientCredits.into()), + 400 if response_body.contains("INVALID_CALLBACK_URL") => { + let message = extract_error_message(response_body) + .unwrap_or_else(|| "callback_url must be a valid, publicly-reachable HTTPS URL".to_string()); + Err(BeebleSpecificApiError::BadWebhookUrl { + message, + webhook_url: context.maybe_callback_url.unwrap_or_default().to_string(), + }.into()) + } + 400 if response_body.contains("VIDEO_TOO_MANY_FRAMES") => { + let message = extract_error_message(response_body) + .unwrap_or_else(|| "Video has too many frames".to_string()); + let (detected, max) = extract_frame_counts(&message); + Err(BeebleSpecificApiError::VideoHasTooManyFrames { + max_frames: max, + detected_frames: detected, + message, + }.into()) + } + 409 => Err(BeebleSpecificApiError::IdempotencyConflict.into()), + 429 => Err(BeebleSpecificApiError::RateLimited.into()), + _ => { + Err(BeebleGenericApiError::UncategorizedBadResponseWithStatusAndBody { + status_code: status, + body: response_body.to_string(), + }.into()) + } + } +} + +/// Try to extract the "message" field from a Beeble error response body. +/// Beeble errors look like: `{"error":{"message":"...","code":"..."}}` +fn extract_error_message(body: &str) -> Option { + let parsed: serde_json::Value = serde_json::from_str(body).ok()?; + parsed.get("error")?.get("message")?.as_str().map(|s| s.to_string()) +} + +/// Parse frame counts from a message like "Video has 419 frames, which exceeds the maximum of 240." +fn extract_frame_counts(message: &str) -> (u64, u64) { + let numbers: Vec = message + .split(|c: char| !c.is_ascii_digit()) + .filter(|s| !s.is_empty()) + .filter_map(|s| s.parse().ok()) + .collect(); + + match numbers.as_slice() { + [detected, max, ..] => (*detected, *max), + [detected] => (*detected, 0), + _ => (0, 0), + } +} diff --git a/crates/api_clients/beeble_client/src/requests/start_generation/mod.rs b/crates/api_clients/beeble_client/src/requests/start_generation/mod.rs new file mode 100644 index 0000000000..47b573c7a4 --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/start_generation/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod handle_error_response; +pub(crate) mod request_types; +pub mod start_generation; diff --git a/crates/api_clients/beeble_client/src/requests/start_generation/request_types.rs b/crates/api_clients/beeble_client/src/requests/start_generation/request_types.rs new file mode 100644 index 0000000000..7f61f0820f --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/start_generation/request_types.rs @@ -0,0 +1,55 @@ +use serde_derive::{Deserialize, Serialize}; + +#[derive(Serialize)] +pub struct StartGenerationRequestBody { + pub generation_type: String, + pub source_uri: String, + pub alpha_mode: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub reference_image_uri: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub alpha_uri: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub max_resolution: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub callback_url: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub idempotency_key: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct GenerationJobResponseBody { + pub id: String, + pub status: String, + pub progress: Option, + pub generation_type: Option, + pub alpha_mode: Option, + pub output: Option, + pub error: Option, + pub created_at: Option, + pub modified_at: Option, + pub completed_at: Option, + pub webhook: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct GenerationOutput { + pub render: Option, + pub source: Option, + pub alpha: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct WebhookStatus { + pub status: Option, + pub attempts: Option, + pub last_error: Option, +} diff --git a/crates/api_clients/beeble_client/src/requests/start_generation/start_generation.rs b/crates/api_clients/beeble_client/src/requests/start_generation/start_generation.rs new file mode 100644 index 0000000000..25ea435002 --- /dev/null +++ b/crates/api_clients/beeble_client/src/requests/start_generation/start_generation.rs @@ -0,0 +1,395 @@ +use log::info; + +use crate::creds::beeble_api_key::BeebleApiKey; +use crate::error::beeble_client_error::BeebleClientError; +use crate::error::beeble_error::BeebleError; +use crate::error::beeble_generic_api_error::BeebleGenericApiError; +use crate::requests::start_generation::handle_error_response::{handle_error_response, ErrorContext}; +use crate::requests::start_generation::request_types::*; + +const BEEBLE_API_BASE_URL: &str = "https://api.beeble.ai/v1"; + +// ── Public types ── + +#[derive(Clone, Debug)] +pub struct StartGenerationArgs { + pub api_key: BeebleApiKey, + pub request: StartGenerationRequest, +} + +#[derive(Clone, Debug)] +pub struct StartGenerationRequest { + /// "image" or "video". + pub generation_type: BeebleGenerationType, + /// Source image/video URI. Supports beeble://, https://, data: base64. + pub source_uri: String, + /// Alpha matte mode. + pub alpha_mode: BeebleAlphaMode, + + /// Text description of desired output (max 2000 chars). + pub prompt: Option, + /// URI for style transfer reference image. + pub reference_image_uri: Option, + /// Custom alpha matte URI. Required for Custom and Select modes. + pub alpha_uri: Option, + /// Maximum output resolution: 720 or 1080. Default 1080. + pub max_resolution: Option, + /// HTTPS webhook URL for completion notification. + pub callback_url: Option, + /// Safe retry key preventing duplicate jobs (max 256 chars). + pub idempotency_key: Option, +} + +#[derive(Copy, Clone, Debug)] +pub enum BeebleGenerationType { + Image, + Video, +} + +#[derive(Copy, Clone, Debug)] +pub enum BeebleAlphaMode { + /// Automatic alpha detection. + Auto, + /// Fill the entire frame. + Fill, + /// Use a custom alpha matte (requires alpha_uri). + Custom, + /// Select regions from the source (requires alpha_uri). + Select, +} + +// ── Public response ── + +#[derive(Debug, Clone)] +pub struct StartGenerationSuccess { + /// Job identifier (swx_...). + pub id: String, + /// in_queue, processing, completed, or failed. + pub status: String, + /// 0-100 progress percentage. + pub progress: Option, + pub generation_type: Option, + pub alpha_mode: Option, + pub output: Option, + pub error: Option, + pub created_at: Option, +} + +#[derive(Debug, Clone)] +pub struct GenerationOutputResult { + /// URL to the rendered output. Expires after 72 hours. + pub render: Option, + /// URL to the source. Expires after 72 hours. + pub source: Option, + /// URL to the alpha matte. Expires after 72 hours. + pub alpha: Option, +} + +// ── Implementation ── + +/// Start a SwitchX compositing generation job. +pub async fn start_generation(args: StartGenerationArgs) -> Result { + let url = format!("{}/switchx/generations", BEEBLE_API_BASE_URL); + let req = args.request; + + info!("Starting Beeble generation: type={:?}, alpha_mode={:?}", req.generation_type, req.alpha_mode); + + let request_body = StartGenerationRequestBody { + generation_type: match req.generation_type { + BeebleGenerationType::Image => "image".to_string(), + BeebleGenerationType::Video => "video".to_string(), + }, + source_uri: req.source_uri, + alpha_mode: match req.alpha_mode { + BeebleAlphaMode::Auto => "auto".to_string(), + BeebleAlphaMode::Fill => "fill".to_string(), + BeebleAlphaMode::Custom => "custom".to_string(), + BeebleAlphaMode::Select => "select".to_string(), + }, + prompt: req.prompt, + reference_image_uri: req.reference_image_uri, + alpha_uri: req.alpha_uri, + max_resolution: req.max_resolution, + callback_url: req.callback_url.clone(), + idempotency_key: req.idempotency_key, + }; + + let client = reqwest::Client::builder() + .build() + .map_err(|err| BeebleClientError::ReqwestClientError(err))?; + + let response = client.post(&url) + .header("x-api-key", &args.api_key.api_key) + .header("Content-Type", "application/json") + .json(&request_body) + .send() + .await + .map_err(|err| BeebleGenericApiError::ReqwestError(err))?; + + let status = response.status(); + + let response_body = response.text() + .await + .map_err(|err| BeebleGenericApiError::ReqwestError(err))?; + + info!("Beeble start generation response: status={}", status); + + if !status.is_success() { + return handle_error_response(status, &response_body, &ErrorContext { + maybe_callback_url: req.callback_url.as_deref(), + }); + } + + let parsed: GenerationJobResponseBody = serde_json::from_str(&response_body) + .map_err(|err| BeebleGenericApiError::SerdeResponseParseError(err, response_body.clone()))?; + + Ok(map_job_response(parsed)) +} + +pub(crate) fn map_job_response(parsed: GenerationJobResponseBody) -> StartGenerationSuccess { + StartGenerationSuccess { + id: parsed.id, + status: parsed.status, + progress: parsed.progress, + generation_type: parsed.generation_type, + alpha_mode: parsed.alpha_mode, + output: parsed.output.map(|o| GenerationOutputResult { + render: o.render, + source: o.source, + alpha: o.alpha, + }), + error: parsed.error, + created_at: parsed.created_at, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::requests::start_generation::request_types::GenerationJobResponseBody; + + #[test] + fn request_body_serializes_minimal() { + let body = StartGenerationRequestBody { + generation_type: "image".to_string(), + source_uri: "beeble://upload_abc/photo.jpg".to_string(), + alpha_mode: "auto".to_string(), + prompt: None, + reference_image_uri: None, + alpha_uri: None, + max_resolution: None, + callback_url: None, + idempotency_key: None, + }; + let json = serde_json::to_string(&body).unwrap(); + assert!(json.contains("\"generation_type\":\"image\"")); + assert!(json.contains("\"alpha_mode\":\"auto\"")); + // Optional fields should not be present + assert!(!json.contains("prompt")); + assert!(!json.contains("reference_image_uri")); + } + + #[test] + fn request_body_serializes_full() { + let body = StartGenerationRequestBody { + generation_type: "video".to_string(), + source_uri: "https://example.com/video.mp4".to_string(), + alpha_mode: "custom".to_string(), + prompt: Some("A sunny beach".to_string()), + reference_image_uri: Some("beeble://upload_ref/ref.jpg".to_string()), + alpha_uri: Some("beeble://upload_alpha/mask.png".to_string()), + max_resolution: Some(1080), + callback_url: Some("https://example.com/webhook".to_string()), + idempotency_key: Some("my-key-123".to_string()), + }; + let json = serde_json::to_string(&body).unwrap(); + assert!(json.contains("\"prompt\":\"A sunny beach\"")); + assert!(json.contains("\"max_resolution\":1080")); + assert!(json.contains("\"idempotency_key\":\"my-key-123\"")); + } + + #[test] + fn response_body_deserializes_queued() { + let json = r#"{ + "id": "swx_abc123", + "status": "in_queue", + "progress": null, + "generation_type": "image", + "alpha_mode": "auto", + "output": null, + "error": null, + "created_at": "2026-05-08T00:00:00Z", + "modified_at": "2026-05-08T00:00:00Z", + "completed_at": null, + "webhook": null + }"#; + let parsed: GenerationJobResponseBody = serde_json::from_str(json).unwrap(); + assert_eq!(parsed.id, "swx_abc123"); + assert_eq!(parsed.status, "in_queue"); + assert!(parsed.output.is_none()); + } + + #[test] + fn response_body_deserializes_completed() { + let json = r#"{ + "id": "swx_abc123", + "status": "completed", + "progress": 100, + "generation_type": "image", + "alpha_mode": "auto", + "output": { + "render": "https://cdn.beeble.ai/render.png", + "source": "https://cdn.beeble.ai/source.png", + "alpha": "https://cdn.beeble.ai/alpha.png" + }, + "error": null, + "created_at": "2026-05-08T00:00:00Z", + "modified_at": "2026-05-08T00:00:01Z", + "completed_at": "2026-05-08T00:00:01Z", + "webhook": null + }"#; + let parsed: GenerationJobResponseBody = serde_json::from_str(json).unwrap(); + assert_eq!(parsed.status, "completed"); + assert_eq!(parsed.progress, Some(100)); + let output = parsed.output.unwrap(); + assert!(output.render.unwrap().contains("render.png")); + } + + #[tokio::test] + #[ignore] // manually test — requires real API key and incurs costs + async fn test_start_image_generation() -> errors::AnyhowResult<()> { + use crate::test_utils::get_test_api_key::get_test_api_key; + + let api_key = get_test_api_key()?; + let result = start_generation(StartGenerationArgs { + api_key, + request: StartGenerationRequest { + generation_type: BeebleGenerationType::Image, + source_uri: "https://images.unsplash.com/photo-1506748686214-e9df14d4d9d0?w=800".to_string(), + alpha_mode: BeebleAlphaMode::Auto, + prompt: Some("A tropical beach at sunset".to_string()), + reference_image_uri: None, + alpha_uri: None, + max_resolution: Some(720), + callback_url: None, + idempotency_key: None, + }, + }).await?; + + println!("Job ID: {}", result.id); + println!("Status: {}", result.status); + println!("Progress: {:?}", result.progress); + assert!(result.id.starts_with("swx_")); + Ok(()) + } + + // ── Helpers for end-to-end tests ── + + async fn download_bytes(url: &str) -> errors::AnyhowResult> { + let response = reqwest::get(url).await?; + let bytes = response.bytes().await?; + Ok(bytes.to_vec()) + } + + fn guess_content_type(filename: &str) -> &'static str { + if filename.ends_with(".mp4") { "video/mp4" } + else if filename.ends_with(".mov") { "video/quicktime" } + else if filename.ends_with(".png") { "image/png" } + else if filename.ends_with(".jpg") || filename.ends_with(".jpeg") { "image/jpeg" } + else if filename.ends_with(".webp") { "image/webp" } + else { "application/octet-stream" } + } + + async fn upload_asset_bytes( + api_key: &crate::creds::beeble_api_key::BeebleApiKey, + filename: &str, + bytes: Vec, + ) -> errors::AnyhowResult { + use crate::requests::create_upload_url::create_upload_url::{ + create_upload_url, CreateUploadUrlArgs, + }; + + // 1. Get presigned upload URL. + let upload = create_upload_url(CreateUploadUrlArgs { + api_key: api_key.clone(), + filename: filename.to_string(), + }).await.map_err(|e| anyhow::anyhow!("create_upload_url failed: {:?}", e))?; + + println!("Upload ID: {}", upload.id); + println!("Upload URL: {}", upload.upload_url); + println!("Beeble URI: {}", upload.beeble_uri); + + // 2. PUT file bytes to the presigned URL with correct Content-Type. + let content_type = guess_content_type(filename); + println!("Uploading {} bytes with Content-Type: {}", bytes.len(), content_type); + + let client = reqwest::Client::new(); + let put_response = client.put(&upload.upload_url) + .header("Content-Type", content_type) + .body(bytes) + .send() + .await?; + + let put_status = put_response.status(); + let put_body = put_response.text().await.unwrap_or_default(); + println!("PUT upload status: {} body: {}", put_status, put_body); + assert!(put_status.is_success(), "PUT upload failed: {} body: {}", put_status, put_body); + + Ok(upload) + } + + #[tokio::test] + #[ignore] // manually test — requires real API key, downloads files, incurs costs + async fn test_end_to_end_video_generation() -> errors::AnyhowResult<()> { + use crate::test_utils::get_test_api_key::get_test_api_key; + use test_data::web::image_urls::FOREST_BACKDROP_IMAGE_URL; + use test_data::web::video_urls::ANGRY_SHIBA_VIDEO_URL; + + let api_key = get_test_api_key()?; + + // 1. Download the source video. + println!("Downloading video..."); + let video_bytes = download_bytes(ANGRY_SHIBA_VIDEO_URL).await?; + println!("Downloaded video: {} bytes", video_bytes.len()); + + // 2. Download the reference image. + println!("Downloading image..."); + let image_bytes = download_bytes(FOREST_BACKDROP_IMAGE_URL).await?; + println!("Downloaded image: {} bytes", image_bytes.len()); + + // 3. Upload the video. + println!("Uploading video..."); + let video_upload = upload_asset_bytes(&api_key, "shiba.mp4", video_bytes).await?; + + // 4. Upload the reference image. + println!("Uploading image..."); + let image_upload = upload_asset_bytes(&api_key, "forest.jpg", image_bytes).await?; + + // 5. Start generation. + println!("Starting generation..."); + let result = start_generation(StartGenerationArgs { + api_key, + request: StartGenerationRequest { + generation_type: BeebleGenerationType::Video, + source_uri: video_upload.beeble_uri, + alpha_mode: BeebleAlphaMode::Auto, + prompt: Some("Place the dog in a forest setting".to_string()), + reference_image_uri: Some(image_upload.beeble_uri), + alpha_uri: None, + max_resolution: Some(720), + callback_url: None, + idempotency_key: None, + }, + }).await.map_err(|e| anyhow::anyhow!("start_generation failed: {:?}", e))?; + + println!("Job ID: {}", result.id); + println!("Status: {}", result.status); + println!("Progress: {:?}", result.progress); + assert!(result.id.starts_with("swx_")); + + // NB: We don't poll to completion here — that would take too long for a test. + // Use get_job_status to poll manually if needed. + + Ok(()) + } +} diff --git a/crates/api_clients/beeble_client/src/test_utils/get_test_api_key.rs b/crates/api_clients/beeble_client/src/test_utils/get_test_api_key.rs new file mode 100644 index 0000000000..34c3465a84 --- /dev/null +++ b/crates/api_clients/beeble_client/src/test_utils/get_test_api_key.rs @@ -0,0 +1,11 @@ +use errors::AnyhowResult; +use std::fs::read_to_string; + +use crate::creds::beeble_api_key::BeebleApiKey; + +#[cfg(test)] +pub fn get_test_api_key() -> AnyhowResult { + let api_key = read_to_string("/Users/bt/Artcraft/credentials/beeble_api_key.txt")?; + let api_key = api_key.trim().to_string(); + Ok(BeebleApiKey::new(api_key)) +} diff --git a/crates/api_clients/beeble_client/src/test_utils/mod.rs b/crates/api_clients/beeble_client/src/test_utils/mod.rs new file mode 100644 index 0000000000..98dab99438 --- /dev/null +++ b/crates/api_clients/beeble_client/src/test_utils/mod.rs @@ -0,0 +1 @@ +pub mod get_test_api_key; diff --git a/crates/api_clients/beeble_client/src/webhook_api/beeble_webhook_payload.rs b/crates/api_clients/beeble_client/src/webhook_api/beeble_webhook_payload.rs new file mode 100644 index 0000000000..e6c155330c --- /dev/null +++ b/crates/api_clients/beeble_client/src/webhook_api/beeble_webhook_payload.rs @@ -0,0 +1,115 @@ +use serde_derive::Deserialize; + +/// Raw deserialized webhook payload from Beeble. +#[derive(Deserialize, Debug, Clone)] +pub struct BeebleWebhookPayload { + pub id: String, + pub status: BeebleWebhookStatus, + pub output: Option, + pub error: Option, + pub completed_at: Option, +} + +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BeebleWebhookStatus { + Completed, + Failed, + /// Catch unknown variants gracefully. + #[serde(other)] + Unknown, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct BeebleWebhookOutput { + pub render: Option, + pub source: Option, + pub alpha: Option, +} + +pub fn parse_beeble_webhook(raw_body: &str) -> Result { + serde_json::from_str(raw_body) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SUCCESS_PAYLOAD: &str = r#"{ + "id": "swx_abc123", + "status": "completed", + "output": { + "render": "https://cdn.beeble.ai/render.mp4", + "source": "https://cdn.beeble.ai/source.mp4", + "alpha": "https://cdn.beeble.ai/alpha.mp4" + }, + "completed_at": "2026-02-23T10:05:00Z" + }"#; + + const FAILURE_PAYLOAD: &str = r#"{ + "id": "swx_abc123", + "status": "failed", + "completed_at": "2026-02-23T10:05:00Z", + "error": "Processing failed" + }"#; + + const UNKNOWN_STATUS_PAYLOAD: &str = r#"{ + "id": "swx_xyz789", + "status": "processing", + "completed_at": null + }"#; + + const NULL_OUTPUT_PAYLOAD: &str = r#"{ + "id": "swx_abc123", + "status": "completed", + "output": null, + "completed_at": "2026-02-23T10:05:00Z" + }"#; + + mod parse_tests { + use super::*; + + #[test] + fn parse_success_payload() { + let payload = parse_beeble_webhook(SUCCESS_PAYLOAD).unwrap(); + assert_eq!(payload.id, "swx_abc123"); + assert_eq!(payload.status, BeebleWebhookStatus::Completed); + assert!(payload.error.is_none()); + + let output = payload.output.unwrap(); + assert_eq!(output.render.as_deref(), Some("https://cdn.beeble.ai/render.mp4")); + assert_eq!(output.source.as_deref(), Some("https://cdn.beeble.ai/source.mp4")); + assert_eq!(output.alpha.as_deref(), Some("https://cdn.beeble.ai/alpha.mp4")); + } + + #[test] + fn parse_failure_payload() { + let payload = parse_beeble_webhook(FAILURE_PAYLOAD).unwrap(); + assert_eq!(payload.id, "swx_abc123"); + assert_eq!(payload.status, BeebleWebhookStatus::Failed); + assert_eq!(payload.error.as_deref(), Some("Processing failed")); + assert!(payload.output.is_none()); + } + + #[test] + fn parse_unknown_status() { + let payload = parse_beeble_webhook(UNKNOWN_STATUS_PAYLOAD).unwrap(); + assert_eq!(payload.id, "swx_xyz789"); + assert_eq!(payload.status, BeebleWebhookStatus::Unknown); + } + + #[test] + fn parse_null_output() { + let payload = parse_beeble_webhook(NULL_OUTPUT_PAYLOAD).unwrap(); + assert_eq!(payload.id, "swx_abc123"); + assert_eq!(payload.status, BeebleWebhookStatus::Completed); + assert!(payload.output.is_none()); + } + + #[test] + fn parse_invalid_json() { + let result = parse_beeble_webhook("not valid json"); + assert!(result.is_err()); + } + } +} diff --git a/crates/api_clients/beeble_client/src/webhook_api/mod.rs b/crates/api_clients/beeble_client/src/webhook_api/mod.rs new file mode 100644 index 0000000000..51fc117576 --- /dev/null +++ b/crates/api_clients/beeble_client/src/webhook_api/mod.rs @@ -0,0 +1 @@ +pub mod beeble_webhook_payload; diff --git a/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/get_inference_job_by_beeble_id.rs b/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/get_inference_job_by_beeble_id.rs new file mode 100644 index 0000000000..7aac61409d --- /dev/null +++ b/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/get_inference_job_by_beeble_id.rs @@ -0,0 +1,134 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use log::warn; +use sqlx::pool::PoolConnection; +use sqlx::{MySql, MySqlPool}; + +use enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty; +use enums::common::job_status_plus::JobStatusPlus; +use errors::AnyhowResult; +use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken; +use tokens::tokens::prompts::PromptToken; +use tokens::tokens::users::UserToken; + + +#[derive(Debug, Default)] +pub struct BeebleJobDetails { + pub job_token: InferenceJobToken, + + pub status: JobStatusPlus, + + pub external_third_party: InferenceJobExternalThirdParty, + pub external_third_party_id: String, + + pub maybe_creator_user_token: Option, + pub maybe_creator_anonymous_visitor_token: Option, + pub creator_ip_address: String, + + pub maybe_prompt_token: Option, + pub maybe_debug_log_event_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, Default)] +struct RawJobRecord { + job_token: InferenceJobToken, + + status: JobStatusPlus, + + // NB: Nullable, but required to be filled for this query + external_third_party: Option, + + // NB: Nullable, but required to be filled for this query + external_third_party_id: Option, + + maybe_creator_user_token: Option, + maybe_creator_anonymous_visitor_token: Option, + creator_ip_address: String, + + maybe_prompt_token: Option, + maybe_debug_log_event_token: Option, + + created_at: DateTime, + updated_at: DateTime, +} + +/// Returns Ok(None) when the record cannot be found. +pub async fn get_inference_job_by_beeble_id(beeble_id: &str, mysql_pool: &MySqlPool) + -> AnyhowResult> +{ + let mut connection = mysql_pool.acquire().await?; + get_inference_job_by_beeble_id_from_connection(beeble_id, &mut connection).await +} + +/// Returns Ok(None) when the record cannot be found. +pub async fn get_inference_job_by_beeble_id_from_connection(beeble_id: &str, mysql_connection: &mut PoolConnection) + -> AnyhowResult> +{ + let maybe_status = sqlx::query_as!( + RawJobRecord, + r#" +SELECT + jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`, + + jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`, + + jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`, + jobs.maybe_external_third_party_id as `external_third_party_id`, + + jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`, + jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`, + jobs.creator_ip_address, + + jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`, + jobs.maybe_debug_log_event_token as `maybe_debug_log_event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`, + + jobs.created_at, + jobs.updated_at + +FROM generic_inference_jobs as jobs + +WHERE jobs.maybe_external_third_party = ? +AND jobs.maybe_external_third_party_id = ? + "#, + InferenceJobExternalThirdParty::Beeble.to_str(), + beeble_id + ) + .fetch_one(&mut **mysql_connection) + .await; + + let record = match maybe_status { + Ok(record) => record, + Err(err) => match err { + sqlx::Error::RowNotFound => return Ok(None), + _ => { + warn!("error querying job record: {:?}", err); + return Err(anyhow!("error querying job record: {:?}", err)); + } + } + }; + + let record = raw_record_to_public_result(record)?; + + Ok(Some(record)) +} + +fn raw_record_to_public_result(record: RawJobRecord) -> AnyhowResult { + Ok(BeebleJobDetails { + job_token: record.job_token, + status: record.status, + external_third_party: record.external_third_party.ok_or_else(|| anyhow!("missing external_third_party"))?, + external_third_party_id: record.external_third_party_id.ok_or_else(|| anyhow!("missing external_third_party_id"))?, + maybe_creator_user_token: record.maybe_creator_user_token, + maybe_creator_anonymous_visitor_token: record.maybe_creator_anonymous_visitor_token, + creator_ip_address: record.creator_ip_address, + maybe_prompt_token: record.maybe_prompt_token, + maybe_debug_log_event_token: record.maybe_debug_log_event_token, + created_at: record.created_at, + updated_at: record.updated_at, + }) +} diff --git a/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/insert_generic_inference_job_for_beeble_queue_with_apriori_job_token.rs b/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/insert_generic_inference_job_for_beeble_queue_with_apriori_job_token.rs new file mode 100644 index 0000000000..affb476a50 --- /dev/null +++ b/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/insert_generic_inference_job_for_beeble_queue_with_apriori_job_token.rs @@ -0,0 +1,168 @@ +use anyhow::anyhow; +use log::info; +use sqlx::{Executor, MySql}; +use std::marker::PhantomData; + +use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty; +use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory; +use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType; +use enums::common::job_status_plus::JobStatusPlus; +use enums::common::visibility::Visibility; +use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken; +use tokens::tokens::prompts::PromptToken; +use tokens::tokens::users::UserToken; + +use crate::errors::database_query_error::DatabaseQueryError; +use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs; + + +pub struct InsertGenericInferenceForBeebleWithAprioriJobTokenArgs<'e, 'c, E> + where E: 'e + Executor<'c, Database = MySql> +{ + pub uuid_idempotency_token: &'e str, + + // NOTE: We'll generate this ahead of time so we can save it with billing info! + pub apriori_job_token: &'e InferenceJobToken, + + /// The external primary key identifier for the job. + pub maybe_external_third_party_id: &'e str, + + pub maybe_inference_args: Option, + + pub maybe_prompt_token: Option<&'e PromptToken>, + + pub maybe_creator_user_token: Option<&'e UserToken>, + pub maybe_avt_token: Option<&'e AnonymousVisitorTrackingToken>, + pub creator_ip_address: &'e str, + pub creator_set_visibility: Visibility, + + pub maybe_debug_log_event_token: Option<&'e DebugLogEventToken>, + + /// Override the initial job status. Defaults to `Pending` when `None`. + /// Set to `Some(JobStatusPlus::CompleteFailure)` for mock/test failure jobs. + pub starting_job_status_override: Option, + + pub maybe_frontend_failure_category: Option, + pub maybe_failure_reason: Option<&'e str>, + + pub mysql_executor: E, + + // TODO: Not sure if this works to tell the compiler we need the lifetime annotation. + // See: https://doc.rust-lang.org/std/marker/struct.PhantomData.html#unused-lifetime-parameters + pub phantom: PhantomData<&'c E>, +} + +pub async fn insert_generic_inference_job_for_beeble_queue_with_apriori_job_token<'e, 'c : 'e, E>( + args: InsertGenericInferenceForBeebleWithAprioriJobTokenArgs<'e, 'c, E> +) -> Result + where E: 'e + Executor<'c, Database = MySql> +{ + let serialized_args_payload = serde_json::ser::to_string(&args.maybe_inference_args) + .map_err(|_e| anyhow!("could not encode inference args"))?; + + const JOB_TYPE: InferenceJobType = InferenceJobType::BeebleQueue; + const INFERENCE_CATEGORY: InferenceCategory = InferenceCategory::VideoGeneration; + const PRODUCT_CATEGORY: InferenceJobProductCategory = InferenceJobProductCategory::BeebleVideo; + const EXTERNAL_THIRD_PARTY: InferenceJobExternalThirdParty = InferenceJobExternalThirdParty::Beeble; + + let status = args.starting_job_status_override.unwrap_or(JobStatusPlus::Pending); + + let maybe_frontend_failure_category_str = args.maybe_frontend_failure_category + .map(|c| c.to_str()); + + let maybe_truncated_failure_reason = args.maybe_failure_reason + .map(|s| if s.len() > 255 { &s[..255] } else { s }); + + let query = sqlx::query!( + r#" +INSERT INTO generic_inference_jobs +SET + token = ?, + uuid_idempotency_token = ?, + + job_type = ?, + + maybe_external_third_party = ?, + maybe_external_third_party_id = ?, + + product_category = ?, + inference_category = ?, + + maybe_model_type = NULL, + maybe_model_token = NULL, + + maybe_input_source_token = NULL, + maybe_input_source_token_type = NULL, + + maybe_download_url = NULL, + maybe_cover_image_media_file_token = NULL, + + maybe_prompt_token = ?, + + maybe_raw_inference_text = NULL, + + maybe_inference_args = ?, + + maybe_creator_user_token = ?, + maybe_creator_anonymous_visitor_token = ?, + creator_ip_address = ?, + creator_set_visibility = ?, + + priority_level = 0, + is_keepalive_required = FALSE, + max_duration_seconds = 0, + + is_debug_request = FALSE, + maybe_routing_tag = NULL, + + maybe_debug_log_event_token = ?, + + frontend_failure_category = ?, + failure_reason = ?, + + status = ? + "#, + args.apriori_job_token.as_str(), + args.uuid_idempotency_token, + + JOB_TYPE.to_str(), + + EXTERNAL_THIRD_PARTY.to_str(), + args.maybe_external_third_party_id, + + PRODUCT_CATEGORY.to_str(), + INFERENCE_CATEGORY.to_str(), + + args.maybe_prompt_token.map(|t| t.to_string()), + + serialized_args_payload, + + args.maybe_creator_user_token.map(|t| t.to_string()), + args.maybe_avt_token.map(|t| t.to_string()), + args.creator_ip_address, + args.creator_set_visibility.to_str(), + + args.maybe_debug_log_event_token.map(|t| t.as_str()), + + maybe_frontend_failure_category_str, + maybe_truncated_failure_reason, + + status.to_str(), + ); + + let query_result = query.execute(args.mysql_executor) + .await; + + let record_id = match query_result { + Err(err) => return Err(DatabaseQueryError::from(err)), + Ok(res) => res.last_insert_id(), + }; + + info!("Insert generic inference job for Beeble queue: {} with record ID {}", args.apriori_job_token, record_id); + + Ok(args.apriori_job_token.clone()) +} diff --git a/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/mod.rs b/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/mod.rs new file mode 100644 index 0000000000..88b852cf5f --- /dev/null +++ b/crates/schema/database/mysql_queries/src/queries/generic_inference/beeble/mod.rs @@ -0,0 +1,2 @@ +pub mod get_inference_job_by_beeble_id; +pub mod insert_generic_inference_job_for_beeble_queue_with_apriori_job_token; diff --git a/crates/schema/database/mysql_queries/src/queries/generic_inference/mod.rs b/crates/schema/database/mysql_queries/src/queries/generic_inference/mod.rs index 43c4342399..a60d53adac 100644 --- a/crates/schema/database/mysql_queries/src/queries/generic_inference/mod.rs +++ b/crates/schema/database/mysql_queries/src/queries/generic_inference/mod.rs @@ -1,3 +1,4 @@ +pub mod beeble; pub mod fal; pub mod job; pub mod moderation; diff --git a/crates/schema/public/enums/src/by_table/debug_logs/debug_log_type.rs b/crates/schema/public/enums/src/by_table/debug_logs/debug_log_type.rs index ff488b446f..cd8ec21f8b 100644 --- a/crates/schema/public/enums/src/by_table/debug_logs/debug_log_type.rs +++ b/crates/schema/public/enums/src/by_table/debug_logs/debug_log_type.rs @@ -22,6 +22,7 @@ pub enum DebugLogType { FalRequest, KinoviRequest, FalWebhook, + BeebleWebhook, } impl_enum_display_and_debug_using_to_str!(DebugLogType); @@ -35,6 +36,7 @@ impl DebugLogType { Self::FalRequest => "fal_request", Self::KinoviRequest => "kinovi_request", Self::FalWebhook => "fal_webhook", + Self::BeebleWebhook => "beeble_webhook", } } @@ -44,6 +46,7 @@ impl DebugLogType { "fal_request" => Ok(Self::FalRequest), "kinovi_request" => Ok(Self::KinoviRequest), "fal_webhook" => Ok(Self::FalWebhook), + "beeble_webhook" => Ok(Self::BeebleWebhook), _ => Err(EnumError::CouldNotConvertFromString(value.to_string())), } } @@ -54,6 +57,7 @@ impl DebugLogType { Self::FalRequest, Self::KinoviRequest, Self::FalWebhook, + Self::BeebleWebhook, ]) } } @@ -74,6 +78,7 @@ mod tests { assert_serialization(DebugLogType::FalRequest, "fal_request"); assert_serialization(DebugLogType::KinoviRequest, "kinovi_request"); assert_serialization(DebugLogType::FalWebhook, "fal_webhook"); + assert_serialization(DebugLogType::BeebleWebhook, "beeble_webhook"); } #[test] @@ -82,6 +87,7 @@ mod tests { assert_eq!(DebugLogType::FalRequest.to_str(), "fal_request"); assert_eq!(DebugLogType::KinoviRequest.to_str(), "kinovi_request"); assert_eq!(DebugLogType::FalWebhook.to_str(), "fal_webhook"); + assert_eq!(DebugLogType::BeebleWebhook.to_str(), "beeble_webhook"); } #[test] @@ -90,6 +96,7 @@ mod tests { assert_eq!(DebugLogType::from_str("fal_request").unwrap(), DebugLogType::FalRequest); assert_eq!(DebugLogType::from_str("kinovi_request").unwrap(), DebugLogType::KinoviRequest); assert_eq!(DebugLogType::from_str("fal_webhook").unwrap(), DebugLogType::FalWebhook); + assert_eq!(DebugLogType::from_str("beeble_webhook").unwrap(), DebugLogType::BeebleWebhook); } #[test] @@ -106,11 +113,12 @@ mod tests { #[test] fn all_variants() { let mut variants = DebugLogType::all_variants(); - assert_eq!(variants.len(), 4); + assert_eq!(variants.len(), 5); assert_eq!(variants.pop_first(), Some(DebugLogType::HttpRequest)); assert_eq!(variants.pop_first(), Some(DebugLogType::FalRequest)); assert_eq!(variants.pop_first(), Some(DebugLogType::KinoviRequest)); assert_eq!(variants.pop_first(), Some(DebugLogType::FalWebhook)); + assert_eq!(variants.pop_first(), Some(DebugLogType::BeebleWebhook)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_external_third_party.rs b/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_external_third_party.rs index ddda651863..6cf1a2351b 100644 --- a/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_external_third_party.rs +++ b/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_external_third_party.rs @@ -12,6 +12,10 @@ use strum::EnumIter; #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum InferenceJobExternalThirdParty { + /// Beeble jobs + #[serde(rename = "beeble")] + Beeble, + /// Fal jobs #[serde(rename = "fal")] #[default] @@ -34,6 +38,7 @@ impl_mysql_enum_coders!(InferenceJobExternalThirdParty); impl InferenceJobExternalThirdParty { pub fn to_str(&self) -> &'static str { match self { + Self::Beeble => "beeble", Self::Fal => "fal", Self::Seedance2Pro => "seedance2pro", Self::Worldlabs => "worldlabs", @@ -42,6 +47,7 @@ impl InferenceJobExternalThirdParty { pub fn from_str(value: &str) -> Result { match value { + "beeble" => Ok(Self::Beeble), "fal" => Ok(Self::Fal), "seedance2pro" => Ok(Self::Seedance2Pro), "worldlabs" => Ok(Self::Worldlabs), @@ -53,6 +59,7 @@ impl InferenceJobExternalThirdParty { // NB: BTreeSet is sorted // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. BTreeSet::from([ + Self::Beeble, Self::Fal, Self::Seedance2Pro, Self::Worldlabs, @@ -70,6 +77,7 @@ mod tests { #[test] fn test_serialization() { + assert_serialization(InferenceJobExternalThirdParty::Beeble, "beeble"); assert_serialization(InferenceJobExternalThirdParty::Fal, "fal"); assert_serialization(InferenceJobExternalThirdParty::Seedance2Pro, "seedance2pro"); assert_serialization(InferenceJobExternalThirdParty::Worldlabs, "worldlabs"); @@ -77,6 +85,7 @@ mod tests { #[test] fn to_str() { + assert_eq!(InferenceJobExternalThirdParty::Beeble.to_str(), "beeble"); assert_eq!(InferenceJobExternalThirdParty::Fal.to_str(), "fal"); assert_eq!(InferenceJobExternalThirdParty::Seedance2Pro.to_str(), "seedance2pro"); assert_eq!(InferenceJobExternalThirdParty::Worldlabs.to_str(), "worldlabs"); @@ -84,6 +93,7 @@ mod tests { #[test] fn from_str() { + assert_eq!(InferenceJobExternalThirdParty::from_str("beeble").unwrap(), InferenceJobExternalThirdParty::Beeble); assert_eq!(InferenceJobExternalThirdParty::from_str("fal").unwrap(), InferenceJobExternalThirdParty::Fal); assert_eq!(InferenceJobExternalThirdParty::from_str("seedance2pro").unwrap(), InferenceJobExternalThirdParty::Seedance2Pro); assert_eq!(InferenceJobExternalThirdParty::from_str("worldlabs").unwrap(), InferenceJobExternalThirdParty::Worldlabs); @@ -92,7 +102,7 @@ mod tests { #[test] fn all_variants() { // Static check - const EXPECTED_COUNT : usize = 3; + const EXPECTED_COUNT : usize = 4; assert_eq!(InferenceJobExternalThirdParty::all_variants().len(), EXPECTED_COUNT); assert_eq!(InferenceJobExternalThirdParty::iter().len(), EXPECTED_COUNT); diff --git a/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs b/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs index b2a6182d4b..755a888238 100644 --- a/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs +++ b/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs @@ -30,6 +30,11 @@ pub enum InferenceJobProductCategory { FalObject, FalBgRemoval, + // =============== BEEBLE =============== + + #[serde(rename = "beeble_video")] + BeebleVideo, + // =============== SEEDANCE 2 PRO =============== #[serde(rename = "seedance2pro_video")] @@ -139,6 +144,7 @@ impl InferenceJobProductCategory { Self::FalVideo => "fal_video", Self::FalObject => "fal_object", Self::FalBgRemoval => "fal_bg_removal", + Self::BeebleVideo => "beeble_video", Self::Seedance2ProVideo => "seedance2pro_video", Self::Seedance2ProCharacter => "seedance2pro_character", Self::WorldlabsSplat => "worldlabs_splat", @@ -174,6 +180,7 @@ impl InferenceJobProductCategory { "fal_video" => Ok(Self::FalVideo), "fal_object" => Ok(Self::FalObject), "fal_bg_removal" => Ok(Self::FalBgRemoval), + "beeble_video" => Ok(Self::BeebleVideo), "seedance2pro_video" => Ok(Self::Seedance2ProVideo), "seedance2pro_character" => Ok(Self::Seedance2ProCharacter), "worldlabs_splat" => Ok(Self::WorldlabsSplat), @@ -212,6 +219,7 @@ impl InferenceJobProductCategory { Self::FalVideo, Self::FalObject, Self::FalBgRemoval, + Self::BeebleVideo, Self::Seedance2ProVideo, Self::Seedance2ProCharacter, Self::WorldlabsSplat, @@ -256,6 +264,7 @@ mod tests { assert_serialization(InferenceJobProductCategory::FalVideo, "fal_video"); assert_serialization(InferenceJobProductCategory::FalObject, "fal_object"); assert_serialization(InferenceJobProductCategory::FalBgRemoval, "fal_bg_removal"); + assert_serialization(InferenceJobProductCategory::BeebleVideo, "beeble_video"); assert_serialization(InferenceJobProductCategory::Seedance2ProVideo, "seedance2pro_video"); assert_serialization(InferenceJobProductCategory::Seedance2ProCharacter, "seedance2pro_character"); assert_serialization(InferenceJobProductCategory::WorldlabsSplat, "worldlabs_splat"); @@ -290,6 +299,7 @@ mod tests { assert_eq!(InferenceJobProductCategory::FalVideo.to_str(), "fal_video"); assert_eq!(InferenceJobProductCategory::FalObject.to_str(), "fal_object"); assert_eq!(InferenceJobProductCategory::FalBgRemoval.to_str(), "fal_bg_removal"); + assert_eq!(InferenceJobProductCategory::BeebleVideo.to_str(), "beeble_video"); assert_eq!(InferenceJobProductCategory::Seedance2ProVideo.to_str(), "seedance2pro_video"); assert_eq!(InferenceJobProductCategory::Seedance2ProCharacter.to_str(), "seedance2pro_character"); assert_eq!(InferenceJobProductCategory::WorldlabsSplat.to_str(), "worldlabs_splat"); @@ -323,6 +333,7 @@ mod tests { assert_eq!(InferenceJobProductCategory::from_str("fal_video").unwrap(), InferenceJobProductCategory::FalVideo); assert_eq!(InferenceJobProductCategory::from_str("fal_object").unwrap(), InferenceJobProductCategory::FalObject); assert_eq!(InferenceJobProductCategory::from_str("fal_bg_removal").unwrap(), InferenceJobProductCategory::FalBgRemoval); + assert_eq!(InferenceJobProductCategory::from_str("beeble_video").unwrap(), InferenceJobProductCategory::BeebleVideo); assert_eq!(InferenceJobProductCategory::from_str("seedance2pro_video").unwrap(), InferenceJobProductCategory::Seedance2ProVideo); assert_eq!(InferenceJobProductCategory::from_str("seedance2pro_character").unwrap(), InferenceJobProductCategory::Seedance2ProCharacter); assert_eq!(InferenceJobProductCategory::from_str("worldlabs_splat").unwrap(), InferenceJobProductCategory::WorldlabsSplat); @@ -352,7 +363,7 @@ mod tests { #[test] fn all_variants() { // Static check - const EXPECTED_COUNT : usize = 30; + const EXPECTED_COUNT : usize = 31; assert_eq!(InferenceJobProductCategory::all_variants().len(), EXPECTED_COUNT); assert_eq!(InferenceJobProductCategory::iter().len(), EXPECTED_COUNT); diff --git a/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_type.rs b/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_type.rs index 9990be8360..cb74bdead9 100644 --- a/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_type.rs +++ b/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_job_type.rs @@ -27,6 +27,10 @@ pub enum InferenceJobType { #[serde(rename = "seedance2pro_character")] Seedance2ProCharacter, + /// Beeble jobs. We poll for results. + #[serde(rename = "beeble_queue")] + BeebleQueue, + /// World Labs jobs. We poll for results. #[serde(rename = "worldlabs_queue")] WorldlabsQueue, @@ -122,6 +126,7 @@ impl InferenceJobType { pub fn to_str(&self) -> &'static str { match self { Self::FalQueue => "fal_queue", + Self::BeebleQueue => "beeble_queue", Self::Seedance2ProQueue => "seedance2pro_queue", Self::Seedance2ProCharacter => "seedance2pro_character", Self::WorldlabsQueue => "worldlabs_queue", @@ -151,6 +156,7 @@ impl InferenceJobType { pub fn from_str(value: &str) -> Result { match value { "fal_queue" => Ok(Self::FalQueue), + "beeble_queue" => Ok(Self::BeebleQueue), "seedance2pro_queue" => Ok(Self::Seedance2ProQueue), "seedance2pro_character" => Ok(Self::Seedance2ProCharacter), "worldlabs_queue" => Ok(Self::WorldlabsQueue), @@ -183,6 +189,7 @@ impl InferenceJobType { // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. BTreeSet::from([ Self::FalQueue, + Self::BeebleQueue, Self::Seedance2ProQueue, Self::Seedance2ProCharacter, Self::WorldlabsQueue, @@ -226,6 +233,7 @@ mod tests { #[test] fn test_serialization() { assert_serialization(InferenceJobType::FalQueue, "fal_queue"); + assert_serialization(InferenceJobType::BeebleQueue, "beeble_queue"); assert_serialization(InferenceJobType::Seedance2ProQueue, "seedance2pro_queue"); assert_serialization(InferenceJobType::Seedance2ProCharacter, "seedance2pro_character"); assert_serialization(InferenceJobType::WorldlabsQueue, "worldlabs_queue"); @@ -254,6 +262,7 @@ mod tests { #[test] fn to_str() { assert_eq!(InferenceJobType::FalQueue.to_str(), "fal_queue"); + assert_eq!(InferenceJobType::BeebleQueue.to_str(), "beeble_queue"); assert_eq!(InferenceJobType::Seedance2ProQueue.to_str(), "seedance2pro_queue"); assert_eq!(InferenceJobType::Seedance2ProCharacter.to_str(), "seedance2pro_character"); assert_eq!(InferenceJobType::WorldlabsQueue.to_str(), "worldlabs_queue"); @@ -282,6 +291,7 @@ mod tests { #[test] fn from_str() { assert_eq!(InferenceJobType::from_str("fal_queue").unwrap(), InferenceJobType::FalQueue); + assert_eq!(InferenceJobType::from_str("beeble_queue").unwrap(), InferenceJobType::BeebleQueue); assert_eq!(InferenceJobType::from_str("seedance2pro_queue").unwrap(), InferenceJobType::Seedance2ProQueue); assert_eq!(InferenceJobType::from_str("seedance2pro_character").unwrap(), InferenceJobType::Seedance2ProCharacter); assert_eq!(InferenceJobType::from_str("worldlabs_queue").unwrap(), InferenceJobType::WorldlabsQueue); @@ -310,7 +320,7 @@ mod tests { #[test] fn all_variants() { // Static check - const EXPECTED_COUNT : usize = 24; + const EXPECTED_COUNT : usize = 25; assert_eq!(InferenceJobType::all_variants().len(), EXPECTED_COUNT); assert_eq!(InferenceJobType::iter().len(), EXPECTED_COUNT); diff --git a/crates/schema/public/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/schema/public/enums/src/by_table/media_files/media_file_origin_product_category.rs index 16824299bf..c54b6447ec 100644 --- a/crates/schema/public/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/schema/public/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -69,6 +69,9 @@ pub enum MediaFileOriginProductCategory { #[serde(rename = "video_gen")] VideoGeneration, + #[serde(rename = "video_edit")] + VideoEdit, + #[serde(rename = "world_gen")] WorldGeneration, @@ -105,6 +108,7 @@ impl MediaFileOriginProductCategory { Self::Mocap => "mocap", Self::ImageGeneration => "image_gen", Self::VideoGeneration => "video_gen", + Self::VideoEdit => "video_edit", Self::WorldGeneration => "world_gen", Self::VideoFilter => "video_filter", Self::Workflow => "workflow", @@ -126,6 +130,7 @@ impl MediaFileOriginProductCategory { "mocap" => Ok(Self::Mocap), "image_gen" => Ok(Self::ImageGeneration), "video_gen" => Ok(Self::VideoGeneration), + "video_edit" => Ok(Self::VideoEdit), "world_gen" => Ok(Self::WorldGeneration), "video_filter" => Ok(Self::VideoFilter), "workflow" => Ok(Self::Workflow), @@ -150,6 +155,7 @@ impl MediaFileOriginProductCategory { Self::Mocap, Self::ImageGeneration, Self::VideoGeneration, + Self::VideoEdit, Self::WorldGeneration, Self::VideoFilter, Self::Workflow, @@ -180,6 +186,7 @@ mod tests { assert_serialization(MediaFileOriginProductCategory::Mocap, "mocap"); assert_serialization(MediaFileOriginProductCategory::ImageGeneration, "image_gen"); assert_serialization(MediaFileOriginProductCategory::VideoGeneration, "video_gen"); + assert_serialization(MediaFileOriginProductCategory::VideoEdit, "video_edit"); assert_serialization(MediaFileOriginProductCategory::WorldGeneration, "world_gen"); assert_serialization(MediaFileOriginProductCategory::VideoFilter, "video_filter"); assert_serialization(MediaFileOriginProductCategory::Workflow, "workflow"); @@ -200,6 +207,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::Mocap.to_str(), "mocap"); assert_eq!(MediaFileOriginProductCategory::ImageGeneration.to_str(), "image_gen"); assert_eq!(MediaFileOriginProductCategory::VideoGeneration.to_str(), "video_gen"); + assert_eq!(MediaFileOriginProductCategory::VideoEdit.to_str(), "video_edit"); assert_eq!(MediaFileOriginProductCategory::WorldGeneration.to_str(), "world_gen"); assert_eq!(MediaFileOriginProductCategory::VideoFilter.to_str(), "video_filter"); assert_eq!(MediaFileOriginProductCategory::Workflow.to_str(), "workflow"); @@ -220,6 +228,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::from_str("mocap").unwrap(), MediaFileOriginProductCategory::Mocap); assert_eq!(MediaFileOriginProductCategory::from_str("image_gen").unwrap(),MediaFileOriginProductCategory::ImageGeneration); assert_eq!(MediaFileOriginProductCategory::from_str("video_gen").unwrap(), MediaFileOriginProductCategory::VideoGeneration); + assert_eq!(MediaFileOriginProductCategory::from_str("video_edit").unwrap(), MediaFileOriginProductCategory::VideoEdit); assert_eq!(MediaFileOriginProductCategory::from_str("world_gen").unwrap(), MediaFileOriginProductCategory::WorldGeneration); assert_eq!(MediaFileOriginProductCategory::from_str("video_filter").unwrap(), MediaFileOriginProductCategory::VideoFilter); assert_eq!(MediaFileOriginProductCategory::from_str("workflow").unwrap(), MediaFileOriginProductCategory::Workflow); @@ -228,7 +237,7 @@ mod tests { #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 16); + assert_eq!(variants.len(), 17); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceFusion)); @@ -242,6 +251,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Mocap)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ImageGeneration)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoGeneration)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoEdit)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::WorldGeneration)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoFilter)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Workflow)); diff --git a/crates/schema/public/enums/src/common/generation/common_model_type.rs b/crates/schema/public/enums/src/common/generation/common_model_type.rs index 7ef7a34a08..1d25d14f93 100644 --- a/crates/schema/public/enums/src/common/generation/common_model_type.rs +++ b/crates/schema/public/enums/src/common/generation/common_model_type.rs @@ -144,6 +144,8 @@ pub enum CommonModelType { Veo3p1, #[serde(rename = "veo_3p1_fast")] Veo3p1Fast, + #[serde(rename = "switch_x")] + SwitchX, // 3D Object generation models #[serde(rename = "hunyuan_3d_2p0")] @@ -224,6 +226,7 @@ impl CommonModelType { Self::Veo3Fast => "veo_3_fast", Self::Veo3p1 => "veo_3p1", Self::Veo3p1Fast => "veo_3p1_fast", + Self::SwitchX => "switch_x", // 3D Object generation models Self::Hunyuan3d2_0 => "hunyuan_3d_2p0", @@ -293,6 +296,7 @@ impl CommonModelType { "veo_3_fast" => Ok(Self::Veo3Fast), "veo_3p1" => Ok(Self::Veo3p1), "veo_3p1_fast" => Ok(Self::Veo3p1Fast), + "switch_x" => Ok(Self::SwitchX), // 3D Object generation models "hunyuan_3d_2p0" => Ok(Self::Hunyuan3d2_0), @@ -366,6 +370,7 @@ impl CommonModelType { Self::Veo3Fast, Self::Veo3p1, Self::Veo3p1Fast, + Self::SwitchX, // 3D Object generation models Self::Hunyuan3d2_0, @@ -436,6 +441,7 @@ impl CommonModelType { Self::Veo3Fast => CommonModelClass::Video, Self::Veo3p1 => CommonModelClass::Video, Self::Veo3p1Fast => CommonModelClass::Video, + Self::SwitchX => CommonModelClass::Video, // 3D Object generation models (mesh) Self::Hunyuan3d2_0 => CommonModelClass::DimensionalMesh, @@ -515,6 +521,7 @@ mod tests { assert_serialization(CommonModelType::Veo3Fast, "veo_3_fast"); assert_serialization(CommonModelType::Veo3p1, "veo_3p1"); assert_serialization(CommonModelType::Veo3p1Fast, "veo_3p1_fast"); + assert_serialization(CommonModelType::SwitchX, "switch_x"); // 3D Object generation models assert_serialization(CommonModelType::Hunyuan3d2_0, "hunyuan_3d_2p0"); assert_serialization(CommonModelType::Hunyuan3d2_1, "hunyuan_3d_2p1"); @@ -581,6 +588,7 @@ mod tests { assert_eq!(CommonModelType::Veo3Fast.to_str(), "veo_3_fast"); assert_eq!(CommonModelType::Veo3p1.to_str(), "veo_3p1"); assert_eq!(CommonModelType::Veo3p1Fast.to_str(), "veo_3p1_fast"); + assert_eq!(CommonModelType::SwitchX.to_str(), "switch_x"); // 3D Object generation models assert_eq!(CommonModelType::Hunyuan3d2_0.to_str(), "hunyuan_3d_2p0"); @@ -647,7 +655,8 @@ mod tests { assert_eq!(CommonModelType::from_str("veo_3_fast").unwrap(), CommonModelType::Veo3Fast); assert_eq!(CommonModelType::from_str("veo_3p1").unwrap(), CommonModelType::Veo3p1); assert_eq!(CommonModelType::from_str("veo_3p1_fast").unwrap(), CommonModelType::Veo3p1Fast); - + assert_eq!(CommonModelType::from_str("switch_x").unwrap(), CommonModelType::SwitchX); + // 3D Object generation models assert_eq!(CommonModelType::from_str("hunyuan_3d_2p0").unwrap(), CommonModelType::Hunyuan3d2_0); assert_eq!(CommonModelType::from_str("hunyuan_3d_2p1").unwrap(), CommonModelType::Hunyuan3d2_1); @@ -660,7 +669,7 @@ mod tests { #[test] fn all_variants() { let mut variants = CommonModelType::all_variants(); - assert_eq!(variants.len(), 57); + assert_eq!(variants.len(), 58); // Image models assert_eq!(variants.pop_first(), Some(CommonModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(CommonModelType::Flux1Schnell)); @@ -715,6 +724,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(CommonModelType::Veo3Fast)); assert_eq!(variants.pop_first(), Some(CommonModelType::Veo3p1)); assert_eq!(variants.pop_first(), Some(CommonModelType::Veo3p1Fast)); + assert_eq!(variants.pop_first(), Some(CommonModelType::SwitchX)); // 3D Object generation models assert_eq!(variants.pop_first(), Some(CommonModelType::Hunyuan3d2_0)); assert_eq!(variants.pop_first(), Some(CommonModelType::Hunyuan3d2_1)); diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 0ddf9a7e34..ad879b10f8 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -50,6 +50,7 @@ datetimes = { path = "../../../lib/data/datetimes" } easyenv.workspace = true elasticsearch_schema = { path = "../../../schema/database/elasticsearch_schema" } enums.workspace = true +beeble_client.workspace = true error_utils.workspace = true errors.workspace = true fal_client.workspace = true diff --git a/crates/service/web/storyteller_web/config/storyteller-web.development.env b/crates/service/web/storyteller_web/config/storyteller-web.development.env index 7ac97ccd2f..e8b3318f5e 100644 --- a/crates/service/web/storyteller_web/config/storyteller-web.development.env +++ b/crates/service/web/storyteller_web/config/storyteller-web.development.env @@ -51,6 +51,8 @@ FAL_WEBHOOK_URL="https://brandons-macbook-pro.taild62114.ts.net/v1/webhooks/fal" #FAL_WEBHOOK_URL="https://halide.tailce84f.ts.net/v1/webhooks/fal" #FAL_WEBHOOK_URL="https://s1.pancake-chimaera.ts.net/api/fal/webhook" +BEEBLE_WEBHOOK_URL="https://brandons-macbook-pro.taild62114.ts.net/v1/webhooks/beeble" + STRIPE_ARTCRAFT_CHECKOUT_SUCCESS_URL="https://getartcraft.com/checkout_success" STRIPE_ARTCRAFT_CHECKOUT_CANCEL_URL="https://getartcraft.com/checkout_cancel" STRIPE_ARTCRAFT_PORTAL_RETURN_URL="https://getartcraft.com/portal_closed" diff --git a/crates/service/web/storyteller_web/src/docs/api_doc.rs b/crates/service/web/storyteller_web/src/docs/api_doc.rs index bf9cf2a95b..41a4880537 100644 --- a/crates/service/web/storyteller_web/src/docs/api_doc.rs +++ b/crates/service/web/storyteller_web/src/docs/api_doc.rs @@ -85,6 +85,7 @@ use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asse use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError; use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::*; use crate::http_server::endpoints::model_download::enqueue_gptsovits_model_download_handler::*; +use artcraft_api_defs::generate::video::edit::beeble_switchx_edit_video::*; use crate::http_server::endpoints::moderation::debug_logs::moderation_list_debug_logs_for_token_handler::*; use crate::http_server::endpoints::moderation::jobs::moderation_get_job_by_token_handler::*; use crate::http_server::endpoints::moderation::user_feature_flags::moderator_edit_user_feature_flags_handler::*; @@ -365,6 +366,7 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux crate::http_server::endpoints::generate::video::image::generate_kling_2_1_master_video_handler::generate_kling_2_1_master_video_handler, crate::http_server::endpoints::generate::video::image::generate_kling_2_1_pro_video_handler::generate_kling_2_1_pro_video_handler, crate::http_server::endpoints::generate::video::image::generate_seedance_1_0_lite_image_to_video_handler::generate_seedance_1_0_lite_image_to_video_handler, + crate::http_server::endpoints::generate::video::edit::beeble_switchx_edit_video_gen_handler::beeble_switchx_edit_video_gen_handler, crate::http_server::endpoints::generate::video::image::generate_veo_2_image_to_video_handler::generate_veo_2_image_to_video_handler, crate::http_server::endpoints::image_studio::upload::upload_snapshot_media_file_handler::upload_snapshot_media_file_handler, crate::http_server::endpoints::inference_job::delete::dismiss_finished_session_jobs_handler::dismiss_finished_session_jobs_handler, @@ -860,6 +862,8 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux GenerateSeedance10LiteImageToVideoResponse, GenerateSeedance10LiteResolution, GenerateVeo2AspectRatio, + BeebleSwitchXEditVideoRequest, + BeebleSwitchXEditVideoResponse, GenerateVeo2Duration, GenerateVeo2ImageToVideoRequest, GenerateVeo2ImageToVideoResponse, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/edit/beeble_switchx_edit_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/edit/beeble_switchx_edit_video_gen_handler.rs new file mode 100644 index 0000000000..60bdf5c98c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/edit/beeble_switchx_edit_video_gen_handler.rs @@ -0,0 +1,345 @@ +use std::sync::Arc; + +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use log::{error, info, warn}; +use sqlx::Acquire; + +use artcraft_api_defs::generate::video::edit::beeble_switchx_edit_video::{ + BeebleSwitchXEditVideoRequest, BeebleSwitchXEditVideoResponse, +}; +use beeble_client::error::beeble_error::BeebleError; +use beeble_client::error::beeble_specific_api_error::BeebleSpecificApiError; +use beeble_client::requests::create_upload_url::upload_bytes::upload_bytes_to_beeble; +use beeble_client::requests::start_generation::start_generation::{ + start_generation, BeebleAlphaMode, BeebleGenerationType, StartGenerationArgs, + StartGenerationRequest, +}; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation::common_generation_mode::CommonGenerationMode; +use enums::common::generation::common_model_type::CommonModelType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::visibility::Visibility; +use http_server_common::request::get_request_ip::get_request_ip; +use mysql_queries::queries::generic_inference::beeble::insert_generic_inference_job_for_beeble_queue_with_apriori_job_token::{ + insert_generic_inference_job_for_beeble_queue_with_apriori_job_token, + InsertGenericInferenceForBeebleWithAprioriJobTokenArgs, +}; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{ + insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem, +}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::http_download_url_to_bytes::http_download_url_to_bytes; +use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map; + +/// Beeble SwitchX Video Edit +#[utoipa::path( + post, + tag = "Generate Videos", + path = "/v1/generate/video/edit/beeble_switchx", + request_body = BeebleSwitchXEditVideoRequest, + responses( + (status = 200, description = "Success", body = BeebleSwitchXEditVideoResponse), + (status = 400, description = "Bad input"), + (status = 401, description = "Unauthorized"), + (status = 402, description = "Payment required"), + (status = 500, description = "Server error"), + ), +)] +pub async fn beeble_switchx_edit_video_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data>, +) -> Result, AdvancedCommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + // ==================== SESSION ==================== // + + let mut mysql_connection = server_state.mysql_pool.acquire().await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + AdvancedCommonWebError::from(e) + })?; + + let user_token = match maybe_user_session.as_ref() { + Some(session) => &session.user_token, + None => return Err(AdvancedCommonWebError::NotAuthorized), + }; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // ==================== VALIDATE INPUT ==================== // + + let source_video_media_token = request.source_video_media_token.as_ref() + .ok_or_else(|| { + AdvancedCommonWebError::BadInputWithSimpleMessage("source_video_media_token is required".to_string()) + })?; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(AdvancedCommonWebError::BadInputWithSimpleMessage(reason)); + } + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + AdvancedCommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + // ==================== LOOKUP MEDIA FILES (BATCH) ==================== // + + let mut tokens_to_lookup: Vec = vec![source_video_media_token.clone()]; + + if let Some(ref_token) = &request.reference_image_media_token { + tokens_to_lookup.push(ref_token.clone()); + } + + let cdn_url_map = lookup_image_urls_as_map( + &http_request, + &mut mysql_connection, + server_state.server_environment, + &tokens_to_lookup, + ).await.map_err(|err| { + warn!("Error looking up media files: {:?}", err); + AdvancedCommonWebError::from(err) + })?; + + let source_video_cdn_url = cdn_url_map.get(source_video_media_token) + .ok_or_else(|| { + warn!("Source video media file not found: {:?}", source_video_media_token); + AdvancedCommonWebError::NotFound + })? + .clone(); + + let maybe_reference_image_cdn_url = request.reference_image_media_token.as_ref() + .map(|ref_token| { + cdn_url_map.get(ref_token) + .cloned() + .ok_or_else(|| { + warn!("Reference image media file not found: {:?}", ref_token); + AdvancedCommonWebError::NotFound + }) + }) + .transpose()?; + + // ==================== DOWNLOAD & UPLOAD TO BEEBLE ==================== // + + let beeble_api_key = &server_state.beeble.api_key; + + // Download + upload source video + info!("Downloading source video from CDN: {}", source_video_cdn_url); + let video_bytes = http_download_url_to_bytes(&source_video_cdn_url).await + .map_err(|err| { + error!("Failed to download source video: {:?}", err); + AdvancedCommonWebError::from_error(err) + })?; + + info!("Downloaded source video: {} bytes", video_bytes.len()); + + let video_extension = std::path::Path::new(&source_video_cdn_url) + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("mp4"); + + let video_filename = format!("{}.{}", source_video_media_token.as_str(), video_extension); + + let video_content_type = match video_extension { + "mp4" => "video/mp4", + "mov" => "video/quicktime", + _ => "application/octet-stream", + }; + + let video_upload = upload_bytes_to_beeble( + beeble_api_key, &video_filename, video_content_type, video_bytes.to_vec(), + ).await.map_err(|err| { + error!("Beeble video upload failed: {:?}", err); + AdvancedCommonWebError::from_error(err) + })?; + + info!("Source video uploaded to Beeble: {}", video_upload.beeble_uri); + + // Download + upload reference image (if provided) + let maybe_reference_beeble_uri = if let Some(ref_cdn_url) = &maybe_reference_image_cdn_url { + info!("Downloading reference image from CDN: {}", ref_cdn_url); + let image_bytes = http_download_url_to_bytes(ref_cdn_url).await + .map_err(|err| { + error!("Failed to download reference image: {:?}", err); + AdvancedCommonWebError::from_error(err) + })?; + + info!("Downloaded reference image: {} bytes", image_bytes.len()); + + let ref_token = request.reference_image_media_token.as_ref().unwrap(); + let image_filename = format!("{}.jpg", ref_token.as_str()); + + let image_upload = upload_bytes_to_beeble( + beeble_api_key, &image_filename, "image/jpeg", image_bytes.to_vec(), + ).await.map_err(|err| { + error!("Beeble image upload failed: {:?}", err); + AdvancedCommonWebError::from_error(err) + })?; + + info!("Reference image uploaded to Beeble: {}", image_upload.beeble_uri); + Some(image_upload.beeble_uri) + } else { + None + }; + + // ==================== CALL BEEBLE GENERATE ==================== // + + let apriori_job_token = InferenceJobToken::generate(); + + let beeble_result = start_generation(StartGenerationArgs { + api_key: beeble_api_key.clone(), + request: StartGenerationRequest { + generation_type: BeebleGenerationType::Video, + source_uri: video_upload.beeble_uri, + alpha_mode: BeebleAlphaMode::Auto, + prompt: request.prompt.clone(), + reference_image_uri: maybe_reference_beeble_uri, + alpha_uri: None, + max_resolution: Some(1080), + callback_url: Some(server_state.beeble.webhook_url.clone()), + idempotency_key: Some(request.uuid_idempotency_token.clone()), + }, + }).await.map_err(|err| { + warn!("Beeble start_generation failed: {:?}", err); + match err { + BeebleError::ApiSpecific(BeebleSpecificApiError::VideoHasTooManyFrames { + max_frames, + detected_frames, + ref message, + }) => { + AdvancedCommonWebError::BadInputWithSimpleMessage(format!( + "Video has too many frames: {} (max: {}); {}", + detected_frames, max_frames, message + )) + }, + _ => { + AdvancedCommonWebError::from_error(err) + } + } + })?; + + let external_job_id = &beeble_result.id; + info!("Beeble job started: id={}, status={}", external_job_id, beeble_result.status); + + // ==================== DB TRANSACTION ==================== // + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection.begin().await.map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + AdvancedCommonWebError::from_error(err) + })?; + + // -- Prompt -- + let prompt_token = match insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: Some(user_token), + maybe_model_type: Some(CommonModelType::SwitchX), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + maybe_generation_mode: Some(CommonGenerationMode::Edit), + maybe_aspect_ratio: None, + maybe_resolution: None, + maybe_batch_count: None, + maybe_generate_audio: None, + maybe_duration_seconds: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None + } + }; + + // -- Prompt context items -- + if let Some(token) = prompt_token.as_ref() { + let mut context_items: Vec = Vec::new(); + + context_items.push(PromptContextItem { + media_token: source_video_media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidRef, + }); + + if let Some(ref_token) = &request.reference_image_media_token { + context_items.push(PromptContextItem { + media_token: ref_token.clone(), + context_semantic_type: PromptContextSemanticType::Imgref, + }); + } + + if !context_items.is_empty() { + if let Err(err) = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: context_items, + transaction: &mut transaction, + }).await { + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + // -- Inference job -- + let db_result = insert_generic_inference_job_for_beeble_queue_with_apriori_job_token( + InsertGenericInferenceForBeebleWithAprioriJobTokenArgs { + apriori_job_token: &apriori_job_token, + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: external_job_id, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: Some(user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + maybe_debug_log_event_token: None, + starting_job_status_override: None, + maybe_frontend_failure_category: None, + maybe_failure_reason: None, + mysql_executor: &mut *transaction, + phantom: Default::default(), + } + ).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + error!("Error inserting inference job: {:?}", err); + return Err(AdvancedCommonWebError::from_error(err)); + } + }; + + transaction.commit().await.map_err(|err| { + error!("Error committing transaction: {:?}", err); + AdvancedCommonWebError::from_error(err) + })?; + + Ok(Json(BeebleSwitchXEditVideoResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/edit/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/edit/mod.rs new file mode 100644 index 0000000000..3810ce6a34 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/edit/mod.rs @@ -0,0 +1 @@ +pub mod beeble_switchx_edit_video_gen_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs index 158c74d131..2f329668b0 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs @@ -1,2 +1,3 @@ +pub mod edit; pub mod image; pub mod multi_function; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/beeble/beeble_webhook_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/beeble/beeble_webhook_handler.rs new file mode 100644 index 0000000000..0c443a9d95 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/beeble/beeble_webhook_handler.rs @@ -0,0 +1,427 @@ +use std::io::Write; +use std::sync::Arc; + +use actix_web::web::Bytes; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use log::{error, info, warn}; +use sqlx::pool::PoolConnection; +use sqlx::MySql; +use tempfile::NamedTempFile; + +use beeble_client::webhook_api::beeble_webhook_payload::{ + parse_beeble_webhook, BeebleWebhookPayload, BeebleWebhookStatus, +}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::debug_logs::debug_log_type::DebugLogType; +use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use ffmpeg_utils::ffprobe::ffprobe_get_info::ffprobe_get_info; +use filesys::path_to_string::path_to_string; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess; +use mysql_queries::queries::debug_logs::insert_debug_log::{insert_debug_log, InsertDebugLogArgs}; +use mysql_queries::queries::generic_inference::beeble::get_inference_job_by_beeble_id::{ + get_inference_job_by_beeble_id_from_connection, BeebleJobDetails, +}; +use mysql_queries::queries::generic_inference::fal::mark_fal_generic_inference_job_successfully_done::{ + mark_fal_generic_inference_job_successfully_done, MarkJobArgs, +}; +use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{ + mark_job_failed_by_token_from_connection, MarkJobFailedByTokenFromConnectionArgs, +}; +use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; +use pager::notification::notification_details_builder::NotificationDetailsBuilder; +use pager::notification::notification_urgency::NotificationUrgency; +use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType}; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError; +use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata; +use crate::state::server_state::ServerState; +use crate::util::http_download_url_to_bytes::http_download_url_to_bytes; + +const VIDEO_BUCKET_PREFIX: Option<&str> = Some("artcraft_"); + +// =============== Handler =============== + +pub async fn beeble_webhook_handler( + http_request: HttpRequest, + request_body_bytes: Bytes, + server_state: web::Data>, +) -> Result, AdvancedCommonWebError> { + + // Step 1: Parse bytes into a UTF-8 string and log it. + let raw_body = String::from_utf8(request_body_bytes.to_vec()) + .map_err(|err| { + error!("Beeble webhook: could not decode request body to UTF-8: {:?}", err); + enqueue_parse_error_alert(&server_state, &http_request, "UTF-8 decode failed", &err, None); + AdvancedCommonWebError::from_error(err) + })?; + + info!("Received Beeble webhook body: {}", raw_body); + + // Step 2: Parse into BeebleWebhookPayload. + let payload = parse_beeble_webhook(&raw_body) + .map_err(|err| { + error!("Beeble webhook: could not parse webhook payload: {:?}", err); + enqueue_parse_error_alert(&server_state, &http_request, "JSON parse failed", &err, Some(&raw_body)); + AdvancedCommonWebError::from_error(err) + })?; + + let job_id = payload.id.as_str(); + + info!("Beeble webhook job_id: {} (status: {:?})", job_id, payload.status); + + // Step 3: Acquire a MySQL connection. + let mut mysql_connection = server_state.mysql_pool.acquire().await?; + + // Step 4: Look up the job by external third-party ID. + let job = match get_inference_job_by_beeble_id_from_connection(job_id, &mut mysql_connection).await { + Ok(Some(record)) => record, + Ok(None) => { + warn!("Could not find job record by Beeble job_id: {}", job_id); + return Err(AdvancedCommonWebError::NotFound); + } + Err(err) => { + error!("Error querying job record for Beeble job_id {}: {:?}", job_id, err); + return Err(AdvancedCommonWebError::from_anyhow_error(err)); + } + }; + + info!("Beeble webhook job record for job_id {}: {:?}", job_id, job); + + // Insert debug log for the webhook payload. + if let Some(debug_log_event_token) = &job.maybe_debug_log_event_token { + if let Err(err) = insert_debug_log(InsertDebugLogArgs { + apriori_debug_log_event_token: Some(debug_log_event_token), + maybe_creator_user_token: job.maybe_creator_user_token.as_ref(), + debug_log_type: DebugLogType::BeebleWebhook, + message: &raw_body, + mysql_executor: &mut *mysql_connection, + phantom: Default::default(), + }).await { + warn!("Failed to insert Beeble webhook debug log: {:?}", err); + } + } + + // Step 5: Branch on status. + let result = match payload.status { + BeebleWebhookStatus::Completed => { + handle_completed(&server_state, &mut mysql_connection, job_id, &payload, &job).await + } + BeebleWebhookStatus::Failed => { + handle_failed(&mut mysql_connection, job_id, &payload, &job).await + } + BeebleWebhookStatus::Unknown => { + warn!("Beeble webhook received unknown status for job_id {}", job_id); + Err(AdvancedCommonWebError::server_error_with_message( + &format!("Beeble webhook received unknown status for job_id {}", job_id), + )) + } + }; + + if let Err(ref err) = result { + if err.is_server_error() { + error!("Beeble webhook error for job_id {}: {:?}", job_id, err); + + let description = format!( + "Beeble webhook failed for job_id: {}\n\nError: {:?}\n\nWebhook JSON Payload: {}", + job_id, err, raw_body, + ); + + let mut builder = if let Some(cause_arc) = err.clone_cause_arc() { + NotificationDetailsBuilder::from_error(cause_arc) + .set_title(format!("Beeble webhook processing failed (job_id: {})", job_id)) + } else { + NotificationDetailsBuilder::from_title( + format!("Beeble webhook processing failed (job_id: {})", job_id) + ) + }; + + builder = builder + .set_description(Some(description)) + .set_third_party_id(Some(job_id.to_string())) + .set_urgency(Some(NotificationUrgency::High)) + .set_http_method(Some(http_request.method().to_string())) + .set_http_path(Some(http_request.path().to_string())); + + if let Err(pager_err) = server_state.pager.enqueue_page(builder.build()) { + error!("Failed to enqueue Beeble webhook pager alert: {:?}", pager_err); + } + } + } + + result +} + +// =============== Private helpers =============== + +async fn handle_completed( + server_state: &ServerState, + mysql_connection: &mut PoolConnection, + job_id: &str, + payload: &BeebleWebhookPayload, + job: &BeebleJobDetails, +) -> Result, AdvancedCommonWebError> { + let output = payload.output.as_ref().ok_or_else(|| { + warn!("Beeble webhook completed but no output for job_id {}", job_id); + AdvancedCommonWebError::server_error_with_message("Beeble webhook completed but output is missing") + })?; + + let render_url = output.render.as_deref().ok_or_else(|| { + warn!("Beeble webhook completed but no render URL for job_id {}", job_id); + AdvancedCommonWebError::server_error_with_message("Beeble webhook completed but render URL is missing") + })?; + + info!("Downloading Beeble render video from {} for job_id {}", render_url, job_id); + + // Download and process the primary render video. + let render_media_token = download_and_upload_video(render_url, job, server_state).await?; + + info!("Render video uploaded with token {} for job_id {}", render_media_token, job_id); + + // Download and process the alpha video if present. + if let Some(ref alpha_url) = output.alpha { + info!("Downloading Beeble alpha video from {} for job_id {}", alpha_url, job_id); + match download_and_upload_video(alpha_url, job, server_state).await { + Ok(alpha_token) => { + info!("Alpha video uploaded with token {} for job_id {}", alpha_token, job_id); + } + Err(err) => { + // Fail open for alpha — the render is the primary artifact. + warn!("Failed to process alpha video for job_id {}: {:?}", job_id, err); + } + } + } + + // Mark the job as successfully done with the render video's media token. + mark_fal_generic_inference_job_successfully_done(MarkJobArgs { + job_token: &job.job_token, + media_file_token: &render_media_token, + maybe_batch_token: None, + mysql_executor: &mut **mysql_connection, + phantom: Default::default(), + }).await.map_err(|err| { + warn!("Error marking job as successfully done for job_id {}: {:?}", job_id, err); + AdvancedCommonWebError::from_anyhow_error(err) + })?; + + info!("Job {} marked as successfully done for Beeble job_id {}", job.job_token, job_id); + + Ok(SimpleGenericJsonSuccess::wrapped(true)) +} + +async fn handle_failed( + mysql_connection: &mut PoolConnection, + job_id: &str, + payload: &BeebleWebhookPayload, + job: &BeebleJobDetails, +) -> Result, AdvancedCommonWebError> { + let public_failure_reason = payload.error + .as_deref() + .unwrap_or("Unknown Beeble error") + .to_string(); + + let internal_failure_reason = format!( + "Beeble job_id={}, error={:?}", + job_id, + payload.error, + ); + + info!( + "Marking job {} as failed for Beeble job_id {}. Reason: {}", + job.job_token.as_str(), + job_id, + public_failure_reason, + ); + + if let Err(err) = mark_job_failed_by_token_from_connection(MarkJobFailedByTokenFromConnectionArgs { + mysql_connection, + job_token: &job.job_token, + maybe_public_failure_reason: Some(&public_failure_reason), + internal_debugging_failure_reason: &internal_failure_reason, + maybe_frontend_failure_category: Some(FrontendFailureCategory::GenerationFailed), + }).await { + error!( + "Error marking job {} as failed for Beeble job_id {}: {:?}", + job.job_token.as_str(), + job_id, + err, + ); + return Err(AdvancedCommonWebError::from_anyhow_error(err)); + } + + info!( + "Job {} marked as failed for Beeble job_id {}.", + job.job_token.as_str(), + job_id, + ); + + Ok(SimpleGenericJsonSuccess::wrapped(true)) +} + +async fn download_and_upload_video( + video_url: &str, + job: &BeebleJobDetails, + server_state: &ServerState, +) -> Result { + // Download with a retry if the first attempt returns suspiciously few bytes. + let mut file_bytes = http_download_url_to_bytes(video_url) + .await + .map_err(|err| { + warn!("Failed to download video from {}: {:?}", video_url, err); + AdvancedCommonWebError::from_error(err) + })?; + + if file_bytes.len() <= 10 { + warn!( + "Downloaded only {} bytes from {} — retrying once", + file_bytes.len(), + video_url, + ); + file_bytes = http_download_url_to_bytes(video_url) + .await + .map_err(|err| { + warn!("Failed to download video on retry from {}: {:?}", video_url, err); + AdvancedCommonWebError::from_error(err) + })?; + } + + // Resolve mime type from magic bytes, with no fallback content_type from Beeble. + let metadata = resolve_file_metadata(&file_bytes, None) + .ok_or_else(|| { + warn!( + "Could not determine file type for Beeble video (bytes: {})", + file_bytes.len(), + ); + AdvancedCommonWebError::server_error_with_message( + &format!("Could not determine file type for Beeble video (bytes: {})", file_bytes.len())) + })?; + + info!("File type: {}, extension: {:?}, source: {:?}", + metadata.mime_type, metadata.file_extension, metadata.source); + + let mime_type = metadata.mime_type.as_str(); + + let media_file_type = MediaFileType::try_from_mime_type(mime_type) + .ok_or_else(|| { + warn!("Unsupported media file type: {}", mime_type); + AdvancedCommonWebError::server_error_with_message( + &format!("Unsupported media file type: {}", mime_type)) + })?; + + let extension_with_period = metadata.file_extension.extension_with_period(); + + let file_size_bytes = file_bytes.len(); + let file_hash = sha256_hash_bytes(&file_bytes) + .map_err(|e| { + warn!("Failed to hash video bytes: {:?}", e); + AdvancedCommonWebError::from_anyhow_error(e) + })?; + + let mut maybe_duration_millis = None; + let mut maybe_frame_width = None; + let mut maybe_frame_height = None; + + if let Ok(mut file) = NamedTempFile::new() { + let _r = file.write_all(&file_bytes); + + if let Ok(video_info) = ffprobe_get_info(&file.path()) { + maybe_duration_millis = video_info.duration + .as_ref() + .map(|duration| duration.millis as u64); + + maybe_frame_width = video_info.dimensions + .as_ref() + .map(|dims| dims.width as u32); + + maybe_frame_height = video_info.dimensions + .as_ref() + .map(|dims| dims.height as u32); + } + } + + let public_upload_path = MediaFileBucketPath::generate_new(VIDEO_BUCKET_PREFIX, Some(&extension_with_period)); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type_process( + public_upload_path.get_full_object_path_str(), + file_bytes.as_ref(), + mime_type) + .await + .map_err(|e| { + warn!("Failed to upload video to bucket: {:?}", e); + AdvancedCommonWebError::from_anyhow_error(e) + })?; + + let media_token = MediaFileInsertBuilder::new() + .maybe_creator_user(job.maybe_creator_user_token.as_ref()) + .maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref()) + .creator_ip_address(&job.creator_ip_address) + .public_bucket_directory_hash(&public_upload_path) + .media_file_class(MediaFileClass::Video) + .media_file_type(media_file_type) + .media_file_origin_category(MediaFileOriginCategory::Inference) + .media_file_origin_product_category(MediaFileOriginProductCategory::VideoEdit) + .mime_type(mime_type) + .file_size_bytes(file_size_bytes as u64) + .maybe_frame_width(maybe_frame_width) + .maybe_frame_height(maybe_frame_height) + .maybe_duration_millis(maybe_duration_millis) + .checksum_sha2(&file_hash) + .maybe_prompt_token(job.maybe_prompt_token.as_ref()) + .insert_pool(&server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Failed to insert video media file record: {:?}", e); + AdvancedCommonWebError::from_error(e) + })?; + + info!("Video media file uploaded with token: {}", media_token); + + // let thumbnail_task_result = + // ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4) + // .with_bucket(server_state.public_bucket_client.bucket_name().as_str()) + // .with_path(&*path_to_string(public_upload_path.to_full_object_pathbuf())) + // .with_output_suffix("thumb") + // .with_event_id(&media_token.to_string()) + // .send_all() + // .await; + // if let Err(err) = thumbnail_task_result { + // // Fail open + // error!("Failed to create some/all thumbnail tasks: {:?}", err); + // } + + Ok(media_token) +} + +/// Send a pager alert for early parse failures (before we have a job_id). +fn enqueue_parse_error_alert( + server_state: &ServerState, + http_request: &HttpRequest, + context: &str, + err: &E, + maybe_raw_body: Option<&str>, +) { + let description = match maybe_raw_body { + Some(body) => format!("Error: {:?}\n\nBeeble Webhook JSON Payload: {}", err, body), + None => format!("Beeble Error: {:?}", err), + }; + + let notification = NotificationDetailsBuilder::from_title( + format!("Beeble webhook parse failure: {}", context)) + .set_description(Some(description)) + .set_urgency(Some(NotificationUrgency::High)) + .set_http_method(Some(http_request.method().to_string())) + .set_http_path(Some(http_request.path().to_string())) + .build(); + + if let Err(pager_err) = server_state.pager.enqueue_page(notification) { + error!("Failed to enqueue Beeble webhook parse error alert: {:?}", pager_err); + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/beeble/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/beeble/mod.rs new file mode 100644 index 0000000000..3452f881cb --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/beeble/mod.rs @@ -0,0 +1 @@ +pub mod beeble_webhook_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/fal_webhook_handler.rs similarity index 95% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/fal_webhook_handler.rs index dd7ee4c683..81d0a98426 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/fal_webhook_handler.rs @@ -1,8 +1,8 @@ use std::sync::Arc; use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError; -use crate::http_server::endpoints::webhooks::process_failure::handle_failed_fal_webhook::handle_failed_fal_webhook; -use crate::http_server::endpoints::webhooks::process_success::handle_successful_fal_webhook::handle_successful_fal_webhook; +use crate::http_server::endpoints::webhooks::fal::process_failure::handle_failed_fal_webhook::handle_failed_fal_webhook; +use crate::http_server::endpoints::webhooks::fal::process_success::handle_successful_fal_webhook::handle_successful_fal_webhook; use crate::state::server_state::ServerState; use actix_web::web::Bytes; use actix_web::web::Json; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/mod.rs new file mode 100644 index 0000000000..98b875c6d9 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/mod.rs @@ -0,0 +1,3 @@ +pub mod fal_webhook_handler; +pub(super) mod process_failure; +pub(super) mod process_success; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/handle_failed_fal_webhook.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_failure/handle_failed_fal_webhook.rs similarity index 100% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/handle_failed_fal_webhook.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_failure/handle_failed_fal_webhook.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_failure/mod.rs similarity index 100% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/mod.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_failure/mod.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/handle_successful_fal_webhook.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/handle_successful_fal_webhook.rs similarity index 100% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/handle_successful_fal_webhook.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/handle_successful_fal_webhook.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/mod.rs similarity index 71% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/mod.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/mod.rs index 0ddb0f004c..5e9447188b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/mod.rs @@ -4,4 +4,4 @@ mod process_images_payload; mod process_model_glb_payload; mod process_model_mesh_payload; mod process_video_payload; -mod resolve_file_metadata; +pub(in crate::http_server::endpoints::webhooks) mod resolve_file_metadata; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_image_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_image_payload.rs similarity index 98% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_image_payload.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_image_payload.rs index a7d0e3f637..db9b252347 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_image_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_image_payload.rs @@ -1,5 +1,5 @@ use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError; -use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata; +use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata; use crate::state::server_state::ServerState; use crate::util::http_download_url_to_bytes::http_download_url_to_bytes; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_images_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_images_payload.rs similarity index 99% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_images_payload.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_images_payload.rs index 25362ea607..0b2e0b7f8d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_images_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_images_payload.rs @@ -1,5 +1,5 @@ use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError; -use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata; +use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata; use crate::state::server_state::ServerState; use crate::util::http_download_url_to_bytes::http_download_url_to_bytes; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_model_glb_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_glb_payload.rs similarity index 99% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_model_glb_payload.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_glb_payload.rs index aba0e58fb4..420c89859c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_model_glb_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_glb_payload.rs @@ -1,5 +1,5 @@ use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError; -use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata; +use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata; use crate::state::server_state::ServerState; use crate::util::http_download_url_to_bytes::http_download_url_to_bytes; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_model_mesh_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_mesh_payload.rs similarity index 97% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_model_mesh_payload.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_mesh_payload.rs index efb18b9a02..1702820ac5 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_model_mesh_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_mesh_payload.rs @@ -1,5 +1,5 @@ use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError; -use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata; +use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata; use crate::state::server_state::ServerState; use crate::util::http_download_url_to_bytes::http_download_url_to_bytes; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_video_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_video_payload.rs similarity index 98% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_video_payload.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_video_payload.rs index 2ea2987639..498786ae3b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_video_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_video_payload.rs @@ -1,5 +1,5 @@ use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError; -use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata; +use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata; use crate::state::server_state::ServerState; use crate::util::http_download_url_to_bytes::http_download_url_to_bytes; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/resolve_file_metadata.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/resolve_file_metadata.rs similarity index 100% rename from crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/resolve_file_metadata.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/resolve_file_metadata.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs index ddd024c0f7..70d0205381 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs @@ -1,3 +1,2 @@ -pub mod fal_webhook_handler; -pub (super) mod process_failure; -pub (super) mod process_success; +pub mod beeble; +pub mod fal; diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs index 9c6bf9b0dc..eb65c0cd10 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs @@ -1,3 +1,4 @@ +use crate::http_server::endpoints::generate::video::edit::beeble_switchx_edit_video_gen_handler::beeble_switchx_edit_video_gen_handler; use crate::http_server::endpoints::generate::cost_estimate::image::estimate_image_cost_handler::estimate_image_cost_handler; use crate::http_server::endpoints::generate::cost_estimate::splat::estimate_splat_cost_handler::estimate_splat_cost_handler; use crate::http_server::endpoints::generate::cost_estimate::video::estimate_video_cost_handler::estimate_video_cost_handler; @@ -218,6 +219,12 @@ where .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service(web::scope("/edit") + .service(web::resource("/beeble_switchx") + .route(web::post().to(beeble_switchx_edit_video_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) .service(web::resource("/kling_1.6_pro_image_to_video") .route(web::post().to(generate_kling_1_6_pro_video_handler)) .route(web::head().to(|| HttpResponse::Ok())) diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/webhook_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/webhook_routes.rs index 1e07afc636..beaddc8d6d 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/webhook_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/webhook_routes.rs @@ -1,4 +1,5 @@ -use crate::http_server::endpoints::webhooks::fal_webhook_handler::fal_webhook_handler; +use crate::http_server::endpoints::webhooks::beeble::beeble_webhook_handler::beeble_webhook_handler; +use crate::http_server::endpoints::webhooks::fal::fal_webhook_handler::fal_webhook_handler; use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; @@ -20,5 +21,9 @@ where .route(web::post().to(fal_webhook_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/beeble") + .route(web::post().to(beeble_webhook_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) } diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index bf2083bac0..b04675f1ce 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -80,7 +80,7 @@ use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator use crate::startup::build_pager::build_pager; use crate::state::certs::google_sign_in_cert::GoogleSignInCert; use crate::state::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; -use crate::state::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, FalData, InMemoryCaches, OpenAiData, ResendData, Seedance2ProData, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, WorldLabsData}; +use crate::state::server_state::{BeebleData, DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, FalData, InMemoryCaches, OpenAiData, ResendData, Seedance2ProData, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, WorldLabsData}; use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; @@ -475,6 +475,12 @@ async fn main() -> AnyhowResult<()> { sort_key_crypto, opaque_cursors: opaque_cursor_encoder, static_api_token_set, + beeble: BeebleData { + api_key: beeble_client::creds::beeble_api_key::BeebleApiKey::new( + easyenv::get_env_string_required("BEEBLE_API_KEY")?, + ), + webhook_url: easyenv::get_env_string_required("BEEBLE_WEBHOOK_URL")?, + }, fal: FalData { api_key: fal_api_key, webhook_url: fal_webhook_url, diff --git a/crates/service/web/storyteller_web/src/state/server_state.rs b/crates/service/web/storyteller_web/src/state/server_state.rs index 1f4c1263d2..f2babed120 100644 --- a/crates/service/web/storyteller_web/src/state/server_state.rs +++ b/crates/service/web/storyteller_web/src/state/server_state.rs @@ -25,6 +25,7 @@ use billing_component::stripe::stripe_config::StripeConfig; use chrono::{DateTime, Utc}; use cloud_storage::bucket_client::BucketClient; use elasticsearch::Elasticsearch; +use beeble_client::creds::beeble_api_key::BeebleApiKey; use fal_client::creds::fal_api_key::FalApiKey; use memory_caching::arc_ttl_sieve::ArcTtlSieve; use memory_caching::single_item_ttl_cache::SingleItemTtlCache; @@ -95,6 +96,8 @@ pub struct ServerState { pub fal: FalData, + pub beeble: BeebleData, + pub seedance2pro: Seedance2ProData, pub openai: OpenAiData, @@ -321,6 +324,13 @@ pub struct FalData { pub webhook_url: String, } +/// Beeble SwitchX integration +#[derive(Clone)] +pub struct BeebleData { + pub api_key: BeebleApiKey, + pub webhook_url: String, +} + /// Seedance 2 Pro integration #[derive(Clone)] pub struct Seedance2ProData { diff --git a/crates/testing/test_data/src/web/mod.rs b/crates/testing/test_data/src/web/mod.rs index 0f0566ead6..6321d56724 100644 --- a/crates/testing/test_data/src/web/mod.rs +++ b/crates/testing/test_data/src/web/mod.rs @@ -1,3 +1,4 @@ pub mod character_tokens; pub mod image_media_tokens; -pub mod image_urls; \ No newline at end of file +pub mod image_urls; +pub mod video_urls; \ No newline at end of file diff --git a/crates/testing/test_data/src/web/video_urls.rs b/crates/testing/test_data/src/web/video_urls.rs new file mode 100644 index 0000000000..93583b2b2e --- /dev/null +++ b/crates/testing/test_data/src/web/video_urls.rs @@ -0,0 +1,5 @@ + + +// media token: m_fz9r14yhfe05ffg9wqda5291p7mp75 +pub const ANGRY_SHIBA_VIDEO_URL : &str = "https://cdn-2.fakeyou.com/media/q/0/1/e/9/q01e9xsye0scamh5mrskr2emsv401x14/artcraft_q01e9xsye0scamh5mrskr2emsv401x14.mp4"; + diff --git a/frontend/apps/artcraft-website/_redirects b/frontend/apps/artcraft-website/_redirects index de13817299..1f8e20f305 100644 --- a/frontend/apps/artcraft-website/_redirects +++ b/frontend/apps/artcraft-website/_redirects @@ -1,3 +1,7 @@ +/background-change /index.html 200 +/background-change/ /index.html 200 +/background-change/* /index.html 200 + /checkout_cancel /index.html 200 /checkout_cancel/ /index.html 200 /checkout_cancel/* /index.html 200 diff --git a/frontend/apps/artcraft-website/src/app/app.tsx b/frontend/apps/artcraft-website/src/app/app.tsx index 0b4fbfbda7..0c21e24c0a 100644 --- a/frontend/apps/artcraft-website/src/app/app.tsx +++ b/frontend/apps/artcraft-website/src/app/app.tsx @@ -13,6 +13,7 @@ import Navbar from "../components/navbar"; import { ToastContainer } from "../components/toast/toast"; import CreateImage from "../pages/create-image"; import CreateVideo from "../pages/create-video"; +import CreateVFX from "../pages/create-vfx"; //import Landing2 from "../pages/landing2"; import Landing3 from "../pages/landing3"; import LandingSD2 from "../pages/landing-sd2"; @@ -54,6 +55,7 @@ export function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/apps/artcraft-website/src/components/navbar/navbar.tsx b/frontend/apps/artcraft-website/src/components/navbar/navbar.tsx index d406e5244b..07fa0a7b86 100644 --- a/frontend/apps/artcraft-website/src/components/navbar/navbar.tsx +++ b/frontend/apps/artcraft-website/src/components/navbar/navbar.tsx @@ -42,6 +42,7 @@ const NAV_ITEMS: NavEntry[] = [ { name: "Home", href: "/" }, { name: "Image", href: "/create-image" }, { name: "Video", href: "/create-video" }, + { name: "BG Change", href: "/background-change" }, { name: "Resources", children: [ @@ -72,6 +73,71 @@ function isEntryActive(pathname: string, entry: NavEntry): boolean { return isPathActive(pathname, entry.href); } +function CreditsChip({ + credits, + align, + onBuyCredits, + onUpgrade, +}: { + credits: number; + align: "center" | "end"; + onBuyCredits: () => void; + onUpgrade: () => void; +}) { + return ( + + } + triggerLabel={ + + {credits.toLocaleString()} + + } + buttonClassName="h-8 px-3 ps-2.5 bg-white/[0.04] hover:bg-white/[0.08] border border-white/[0.08] shadow-none text-white/80 rounded-lg gap-1.5" + panelClassName="mt-2 bg-[#1a1a1a] border border-white/[0.08] text-white rounded-xl" + > + {(close) => ( +
+
+ + Your credit balance + + +
+
+ + {credits.toLocaleString()} +
+
+ +
+
+ )} + + ); +} + async function fetchCredits(): Promise { try { const api = new CreditsApi(); @@ -281,62 +347,12 @@ export default function Navbar() { {credits !== null && ( - - } - triggerLabel={ - - {credits.toLocaleString()} - - } - buttonClassName="h-8 px-3 ps-2.5 bg-white/[0.04] hover:bg-white/[0.08] border border-white/[0.08] shadow-none text-white/80 rounded-lg gap-1.5" - panelClassName="mt-2 bg-[#1a1a1a] border border-white/[0.08] text-white rounded-xl" - > - {(close) => ( -
-
- - Your credit balance - - -
-
- - {credits.toLocaleString()} -
-
- -
-
- )} - + onBuyCredits={() => setCreditsModalOpen(true)} + onUpgrade={() => navigate("/pricing")} + /> )} {hasPaidPlan === false && ( @@ -488,8 +504,16 @@ export default function Navbar() { )} - {/* Mobile: hamburger + task queue */} -
+ {/* Mobile: credits + task queue + hamburger */} +
+ {user && credits !== null && ( + setCreditsModalOpen(true)} + onUpgrade={() => navigate("/pricing")} + /> + )} {user && } Open main menu @@ -640,15 +664,6 @@ export default function Navbar() { /> Discord - {credits !== null && ( - - - {credits.toLocaleString()} - - )}
@@ -468,9 +468,7 @@ function getPrompt(job: Job, promptsMap?: Map): string { ? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken)) : undefined; return ( - cached?.maybe_positive_prompt || - job.request.maybe_raw_inference_text || - "" + cached?.maybe_positive_prompt || job.request.maybe_raw_inference_text || "" ); } @@ -483,8 +481,7 @@ function formatTitleParts(job: Job, promptsMap?: Map) { const taskTypeStr = job.request.inference_category?.toLowerCase() ?? ""; const modelTypeStr = cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? ""; - const providerKey = - cachedPrompt?.maybe_generation_provider ?? modelTypeStr; + const providerKey = cachedPrompt?.maybe_generation_provider ?? modelTypeStr; let kind: string | undefined; if (taskTypeStr.includes("video")) { @@ -532,18 +529,24 @@ function jobsToInProgress( const parts = formatTitleParts(j, promptsMap); const isVideo = j.request.inference_category?.toLowerCase().includes("video") ?? false; - const modelType = j.request.maybe_model_type; - // Look up per-model estimated duration, cache per task + const promptToken = j.request.maybe_prompt_token; + const cachedPrompt = promptToken + ? (promptsMap.get(promptToken) ?? getCachedPrompt(promptToken)) + : undefined; + const modelType = + cachedPrompt?.maybe_model_type ?? j.request.maybe_model_type; + + // Look up per-model estimated duration from the model list, cache per + // task. Cache only once `modelType` is resolved so a transient + // "prompt not loaded yet" render doesn't lock in the wrong fallback. let duration = taskDurationCache.get(j.job_token); - if (!duration) { - const model = modelType - ? ALL_MODELS_LIST.find( - (m) => m.tauriId === modelType || m.id === modelType, - ) - : undefined; + if (duration === undefined) { + const model = findModelByKey(modelType); duration = model?.progressBarTime ?? (isVideo ? 900000 : 30000); - taskDurationCache.set(j.job_token, duration); + if (modelType) { + taskDurationCache.set(j.job_token, duration); + } } // Always use time-based fake progress (matching desktop app behavior) @@ -663,7 +666,8 @@ export const TaskQueue = () => { const promptTokens = useMemo(() => { const tokens: string[] = []; for (const j of jobs) { - if (j.request.maybe_prompt_token) tokens.push(j.request.maybe_prompt_token); + if (j.request.maybe_prompt_token) + tokens.push(j.request.maybe_prompt_token); } return tokens; }, [jobs]); @@ -710,7 +714,7 @@ export const TaskQueue = () => { primaryActionText: "", primaryActionIcon: faTrashAlt, primaryActionBtnClassName: "", - onConfirm: async () => { }, + onConfirm: async () => {}, }); // ── Confirmation handlers ────────────────────────────────────────── @@ -814,7 +818,9 @@ export const TaskQueue = () => { const failedJobs = fetchedJobs.filter((j) => FAILED_STATUSES.has(j.status.status), ); - const newCompletedIdSet = new Set(completedJobs.map((j) => j.job_token)); + const newCompletedIdSet = new Set( + completedJobs.map((j) => j.job_token), + ); const newFailedIdSet = new Set(failedJobs.map((j) => j.job_token)); if (!initialLoadDoneRef.current) { diff --git a/frontend/apps/artcraft-website/src/components/prompt-box/upload-image.ts b/frontend/apps/artcraft-website/src/components/prompt-box/upload-image.ts index 019fba4e2b..28fad4b549 100644 --- a/frontend/apps/artcraft-website/src/components/prompt-box/upload-image.ts +++ b/frontend/apps/artcraft-website/src/components/prompt-box/upload-image.ts @@ -1,4 +1,4 @@ -import { MediaUploadApi } from "@storyteller/api"; +import { EIntermediateFile, MediaUploadApi } from "@storyteller/api"; import { UploaderState, UploaderStates } from "@storyteller/common"; export type UploadImageFn = (args: { @@ -21,6 +21,9 @@ export const uploadImage: UploadImageFn = async ({ blob: assetFile, fileName: assetFile.name || `reference-${Date.now()}`, maybe_title: `ref_image_${title}`, + // User-uploaded reference images should appear in their library, not be + // hidden as intermediate system files. + is_intermediate_system_file: EIntermediateFile.false, }); if (!response?.success || !response.data) { diff --git a/frontend/apps/artcraft-website/src/main.tsx b/frontend/apps/artcraft-website/src/main.tsx index 7f42250f27..a16f3fc255 100644 --- a/frontend/apps/artcraft-website/src/main.tsx +++ b/frontend/apps/artcraft-website/src/main.tsx @@ -18,7 +18,7 @@ if (import.meta.env.DEV) { window.location.origin, ); // NB: This is for Brandon to test with storyteller-web locally: - StorytellerApiHostStore.getInstance().setDevelopment(); + //StorytellerApiHostStore.getInstance().setDevelopment(); } catch (e) { console.warn("Failed to set dev API host override", e); } diff --git a/frontend/apps/artcraft-website/src/pages/create-vfx/create-vfx.tsx b/frontend/apps/artcraft-website/src/pages/create-vfx/create-vfx.tsx new file mode 100644 index 0000000000..df5ea2bab0 --- /dev/null +++ b/frontend/apps/artcraft-website/src/pages/create-vfx/create-vfx.tsx @@ -0,0 +1,400 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { Link } from "react-router-dom"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faSpinnerThird, faSparkles } from "@fortawesome/pro-solid-svg-icons"; +import { Button } from "@storyteller/ui-button"; +import { + PromptBoxVFX, + TruchetPattern, + VFXResultCard, + useVFXStore, +} from "@storyteller/ui-vfx"; +import Seo from "../../components/seo"; +import { useAuthCheck } from "../../components/generation-gallery"; +import { toast } from "../../components/toast/toast"; +import { uploadImage } from "../../components/prompt-box/upload-image"; +import { uploadVideo } from "../../components/prompt-box/upload-media"; +import { + enqueueBackgroundChangeGeneration, + listSessionBackgroundChangeJobs, + startBackgroundChangePolling, +} from "./generate-background-change-api"; + +export default function CreateVFX() { + const { user, authChecked } = useAuthCheck(); + + const history = useVFXStore((s) => s.history); + const startResult = useVFXStore((s) => s.startResult); + const attachJobToken = useVFXStore((s) => s.attachJobToken); + const completeResult = useVFXStore((s) => s.completeResult); + const failResult = useVFXStore((s) => s.failResult); + const dismissResult = useVFXStore((s) => s.dismissResult); + const seedFromSession = useVFXStore((s) => s.seedFromSession); + const updateMediaForResult = useVFXStore((s) => s.updateMediaForResult); + const source = useVFXStore((s) => s.source); + const reference = useVFXStore((s) => s.reference); + const prompt = useVFXStore((s) => s.prompt); + + const promptBoxRef = useRef(null); + const [promptBoxHeight, setPromptBoxHeight] = useState(96); + const [isSubmitting, setIsSubmitting] = useState(false); + const pollersRef = useRef(new Map void>()); + + useEffect(() => { + const el = promptBoxRef.current; + if (!el || typeof ResizeObserver === "undefined") return; + const update = () => setPromptBoxHeight(el.offsetHeight); + update(); + const ro = new ResizeObserver(update); + ro.observe(el); + return () => ro.disconnect(); + }, []); + + useEffect(() => { + const pollers = pollersRef.current; + return () => { + pollers.forEach((cancel) => cancel()); + pollers.clear(); + }; + }, []); + + // On mount, reconcile against the server: pull recent SwitchX jobs from the + // session so the page shows in-progress / completed / failed cards across + // refreshes and devices, not just from localStorage. Then resume polling + // for anything that's still pending. + useEffect(() => { + if (!user) return; + + let cancelled = false; + const pollers = pollersRef.current; + + const startPollFor = ( + resultId: string, + jobToken: string, + ) => { + if (pollers.has(resultId)) return; + const cancel = startBackgroundChangePolling( + jobToken, + (output) => { + completeResult(resultId, output.cdn_url); + pollers.delete(resultId); + window.dispatchEvent(new Event("task-queue-update")); + }, + (reason) => { + failResult(resultId, reason); + pollers.delete(resultId); + window.dispatchEvent(new Event("task-queue-update")); + }, + ); + pollers.set(resultId, cancel); + }; + + (async () => { + try { + const sessionJobs = await listSessionBackgroundChangeJobs(); + if (cancelled) return; + + const existing = useVFXStore.getState().history; + const existingByToken = new Map( + existing + .filter((r) => r.inferenceJobToken) + .map((r) => [r.inferenceJobToken!, r] as const), + ); + + for (const job of sessionJobs) { + const found = existingByToken.get(job.jobToken); + if (found) { + // Server may know more than we do (e.g. completed while we were + // away). Reconcile terminal states. + if (found.status === "pending" && job.status === "complete" && job.outputUrl) { + completeResult(found.id, job.outputUrl); + } else if (found.status === "pending" && job.status === "failed") { + failResult(found.id, job.failureReason ?? "Generation failed"); + } + // Patch in CDN media + prompt from the batch-prompt response so + // dead blob: URLs (from a prior session) are replaced. + updateMediaForResult(found.id, { + source: job.source + ? { + id: job.source.mediaToken, + url: job.source.url, + mediaToken: job.source.mediaToken, + } + : undefined, + reference: job.reference + ? { + id: job.reference.mediaToken, + url: job.reference.url, + mediaToken: job.reference.mediaToken, + } + : undefined, + prompt: job.prompt, + }); + continue; + } + // Missing locally — seed it. + const id = + typeof crypto !== "undefined" && crypto.randomUUID + ? crypto.randomUUID() + : Math.random().toString(36).slice(2); + seedFromSession({ + id, + status: job.status, + prompt: job.prompt, + resolution: "720p", + source: job.source + ? { + id: job.source.mediaToken, + url: job.source.url, + mediaToken: job.source.mediaToken, + } + : undefined, + reference: job.reference + ? { + id: job.reference.mediaToken, + url: job.reference.url, + mediaToken: job.reference.mediaToken, + } + : undefined, + inferenceJobToken: job.jobToken, + outputUrl: job.outputUrl, + failureReason: job.failureReason, + createdAt: job.createdAt, + }); + } + + if (cancelled) return; + + // Resume polling for everything still pending (local + freshly seeded). + const pending = useVFXStore + .getState() + .history.filter((r) => r.status === "pending" && r.inferenceJobToken); + for (const item of pending) { + startPollFor(item.id, item.inferenceJobToken!); + } + } catch { + // Network errors during session reconciliation are non-fatal — fall + // back to local-only behavior by polling whatever localStorage had. + if (cancelled) return; + const pending = useVFXStore + .getState() + .history.filter((r) => r.status === "pending" && r.inferenceJobToken); + for (const item of pending) { + startPollFor(item.id, item.inferenceJobToken!); + } + } + })(); + + return () => { + cancelled = true; + }; + }, [user, completeResult, failResult, seedFromSession, updateMediaForResult]); + + const handleSubmit = useCallback(async () => { + if (!source || !reference || isSubmitting) return; + setIsSubmitting(true); + const id = startResult(); + const enqueueResult = await enqueueBackgroundChangeGeneration({ + sourceVideoMediaToken: source.mediaToken, + referenceImageMediaToken: reference.mediaToken, + prompt: prompt.trim() || null, + }); + setIsSubmitting(false); + + if (!enqueueResult.success) { + const message = enqueueResult.backendUnavailable + ? "Background change backend coming soon. Your inputs are saved." + : enqueueResult.error; + failResult(id, message); + if (enqueueResult.backendUnavailable) { + toast.success(message); + } else { + toast.error(message); + } + return; + } + + attachJobToken(id, enqueueResult.jobToken); + window.dispatchEvent(new Event("task-queue-update")); + const cancel = startBackgroundChangePolling( + enqueueResult.jobToken, + (output) => { + completeResult(id, output.cdn_url); + pollersRef.current.delete(id); + window.dispatchEvent(new Event("task-queue-update")); + }, + (reason) => { + failResult(id, reason); + pollersRef.current.delete(id); + window.dispatchEvent(new Event("task-queue-update")); + }, + ); + pollersRef.current.set(id, cancel); + }, [ + source, + reference, + prompt, + isSubmitting, + startResult, + attachJobToken, + completeResult, + failResult, + ]); + + if (!authChecked) { + return ( +
+ +
+ ); + } + + if (!user) { + return ( +
+ +
+
+
+
+ +

Background Change

+

+ Sign in to swap the backdrop of a video using a reference image. +

+
+ + + + + + +
+
+
+ ); + } + + return ( +
+ + + {/* Truchet pattern is fixed to the viewport so it stays put as the + page scrolls behind the (also fixed) navbar. */} +
+ +
+ + {history.length === 0 ? ( +
+ +
+ ) : ( +
+
+ {history.map((r) => ( + dismissResult(r.id)} + className="w-[min(960px,calc(100vw-32px))]" + /> + ))} +
+
+ )} + +
+ +
+
+ toast.error(msg)} + hideResolution + /> +
+
+
+ ); +} + +interface EmptyStateProps { + title: string; + subtitle: string; +} + +const EmptyState = ({ title, subtitle }: EmptyStateProps) => ( +
+
+ +
+

{title}

+

{subtitle}

+
+); diff --git a/frontend/apps/artcraft-website/src/pages/create-vfx/generate-background-change-api.ts b/frontend/apps/artcraft-website/src/pages/create-vfx/generate-background-change-api.ts new file mode 100644 index 0000000000..b9a9a2f367 --- /dev/null +++ b/frontend/apps/artcraft-website/src/pages/create-vfx/generate-background-change-api.ts @@ -0,0 +1,310 @@ +import { JobsApi, PromptsApi } from "@storyteller/api"; +import { + newIdempotencyToken, + submitVFXJob, + VFX_NOT_AVAILABLE_ERROR, +} from "@storyteller/ui-vfx"; + +export interface GeneratedBackgroundChange { + cdn_url: string; +} + +// ── Request params ─────────────────────────────────────────────────────── + +export interface GenerateBackgroundChangeParams { + sourceVideoMediaToken: string; + referenceImageMediaToken?: string | null; + prompt?: string | null; +} + +export type EnqueueResult = + | { success: true; jobToken: string } + | { success: false; error: string; backendUnavailable?: boolean }; + +// ── Enqueue generation ─────────────────────────────────────────────────── + +export async function enqueueBackgroundChangeGeneration( + params: GenerateBackgroundChangeParams, +): Promise { + const trimmedPrompt = params.prompt?.trim(); + const response = await submitVFXJob({ + source_video_media_token: params.sourceVideoMediaToken, + reference_image_media_token: params.referenceImageMediaToken ?? null, + prompt: trimmedPrompt && trimmedPrompt.length > 0 ? trimmedPrompt : null, + uuid_idempotency_token: newIdempotencyToken(), + }); + + if (response.success) { + return { success: true, jobToken: response.inference_job_token }; + } + + return { + success: false, + error: response.error_message ?? "Failed to submit background change job.", + backendUnavailable: response.error_code_str === VFX_NOT_AVAILABLE_ERROR, + }; +} + +// ── Poll for completion ────────────────────────────────────────────────── + +export async function pollBackgroundChangeJobResult( + jobToken: string, +): Promise<{ + status: "pending" | "complete" | "failed"; + output?: GeneratedBackgroundChange; + error?: string; +}> { + const jobsApi = new JobsApi(); + const response = await jobsApi.GetJobByToken({ token: jobToken }); + + if (!response.success || !response.data) { + return { status: "pending" }; + } + + const state = response.data; + const statusStr = state.status?.status?.toLowerCase() ?? ""; + + if (statusStr === "complete_success" || statusStr === "complete") { + const result = state.maybe_result as Record | undefined; + const cdnUrl = + ((result as any)?.media_links?.cdn_url as string | undefined) ?? + ((result as any)?.cdn_url as string | undefined) ?? + undefined; + + if (cdnUrl) { + return { status: "complete", output: { cdn_url: cdnUrl } }; + } + return { + status: "failed", + error: "Generation finished but returned no output URL.", + }; + } + + if ( + statusStr.includes("fail") || + statusStr.includes("error") || + statusStr === "dead" + ) { + return { + status: "failed", + error: + state.status?.maybe_failure_message ?? + state.status?.maybe_extra_status_description ?? + "Generation failed", + }; + } + + return { status: "pending" }; +} + +// ── Session sync ───────────────────────────────────────────────────────── + +/** + * Heuristic match for any Beeble SwitchX model variant the backend may emit. + * Matches: `switch_x`, `switchx`, `beeble_switchx`, `beeble_switch_x`, + * `Beeble.SwitchX`, etc. + */ +const isSwitchXModel = (modelType?: string | null): boolean => { + if (!modelType) return false; + const normalized = modelType.toLowerCase().replace(/\./g, "_"); + return normalized.includes("switchx") || normalized.includes("switch_x"); +}; + +export interface SessionMediaRef { + url: string; + mediaToken: string; +} + +export interface SessionBackgroundChangeJob { + jobToken: string; + status: "pending" | "complete" | "failed"; + prompt: string; + source?: SessionMediaRef; + reference?: SessionMediaRef; + outputUrl?: string; + failureReason?: string; + createdAt: number; +} + +/** + * Fetch the user's recent jobs from `/v1/jobs/session` and filter down to + * Beeble SwitchX (background-change) jobs. Used on mount so the page reflects + * server-side state across refreshes and devices, not just localStorage. + * + * The model type may live on `job.request.maybe_model_type` OR on the prompt + * (looked up by `maybe_prompt_token`) — that's the same fallback chain the + * task queue / video page use, so we match here too. + */ +export async function listSessionBackgroundChangeJobs(): Promise< + SessionBackgroundChangeJob[] +> { + const jobsApi = new JobsApi(); + const response = await jobsApi.ListRecentJobs(); + if (!response.success || !response.data) return []; + + // Resolve any missing model_types via the prompts API. + const promptTokens = Array.from( + new Set( + response.data + .map((j) => j.request.maybe_prompt_token) + .filter((t): t is string => !!t), + ), + ); + const promptsByToken = new Map< + string, + { + maybe_model_type?: string | null; + maybe_positive_prompt?: string | null; + maybe_context_images?: Array<{ + media_token: string; + media_links: { + cdn_url: string; + maybe_video_previews?: unknown; + }; + semantic: string; + }> | null; + } + >(); + if (promptTokens.length > 0) { + try { + const promptsApi = new PromptsApi(); + const promptsResponse = await promptsApi.BatchGetPrompts({ + tokens: promptTokens, + }); + if (promptsResponse.success && promptsResponse.data) { + for (const p of promptsResponse.data) { + promptsByToken.set(p.token, p); + } + } + } catch { + // Non-fatal: fall back to job.request.maybe_model_type only. + } + } + + const out: SessionBackgroundChangeJob[] = []; + for (const job of response.data) { + const promptToken = job.request.maybe_prompt_token; + const cachedPrompt = promptToken + ? promptsByToken.get(promptToken) + : undefined; + const modelType = + cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type; + if (!isSwitchXModel(modelType)) continue; + + const statusStr = job.status?.status?.toLowerCase() ?? ""; + let status: SessionBackgroundChangeJob["status"]; + let outputUrl: string | undefined; + let failureReason: string | undefined; + + if (statusStr === "complete_success" || statusStr === "complete") { + status = "complete"; + const result = job.maybe_result as unknown as + | Record + | undefined; + outputUrl = + ((result as any)?.media_links?.cdn_url as string | undefined) ?? + ((result as any)?.cdn_url as string | undefined); + } else if ( + statusStr.includes("fail") || + statusStr.includes("error") || + statusStr === "dead" || + statusStr.includes("cancel") + ) { + status = "failed"; + failureReason = + job.status?.maybe_failure_message ?? + job.status?.maybe_extra_status_description ?? + "Generation failed"; + } else { + status = "pending"; + } + + // Extract source video + reference image from the prompt's context images. + // We don't depend on exact semantic strings — videos are the items whose + // media_links carry video previews, images are everything else. + let source: SessionMediaRef | undefined; + let reference: SessionMediaRef | undefined; + const ctxImages = cachedPrompt?.maybe_context_images ?? []; + for (const ci of ctxImages) { + const isVideo = !!ci.media_links?.maybe_video_previews; + if (isVideo && !source) { + source = { url: ci.media_links.cdn_url, mediaToken: ci.media_token }; + } else if (!isVideo && !reference) { + reference = { + url: ci.media_links.cdn_url, + mediaToken: ci.media_token, + }; + } + if (source && reference) break; + } + + const resolvedPrompt = + cachedPrompt?.maybe_positive_prompt ?? + job.request.maybe_raw_inference_text ?? + ""; + + out.push({ + jobToken: job.job_token, + status, + prompt: resolvedPrompt, + source, + reference, + outputUrl, + failureReason, + createdAt: new Date(job.created_at).getTime(), + }); + } + + return out; +} + +// ── Polling controller ─────────────────────────────────────────────────── + +export function startBackgroundChangePolling( + jobToken: string, + onComplete: (output: GeneratedBackgroundChange) => void, + onError: (reason: string) => void, + intervalMs = 4000, + maxAttempts = 75, // ~5 min at 4s — Beeble SwitchX expected runtime +): () => void { + let attempts = 0; + let stopped = false; + + const poll = async () => { + if (stopped) return; + attempts++; + + try { + const result = await pollBackgroundChangeJobResult(jobToken); + if (stopped) return; + + if (result.status === "complete" && result.output) { + onComplete(result.output); + return; + } + if (result.status === "failed") { + onError(result.error ?? "Generation failed"); + return; + } + if (attempts >= maxAttempts) { + onError("Generation timed out"); + return; + } + + setTimeout(poll, intervalMs); + } catch { + if (!stopped && attempts < maxAttempts) { + setTimeout(poll, intervalMs * 2); + } else if (!stopped) { + onError("Network error during polling"); + } + } + }; + + // Start first poll after a short delay (video jobs take time to start) + setTimeout(poll, 3000); + + return () => { + stopped = true; + }; +} diff --git a/frontend/apps/artcraft-website/src/pages/create-vfx/index.ts b/frontend/apps/artcraft-website/src/pages/create-vfx/index.ts new file mode 100644 index 0000000000..3bb160f75a --- /dev/null +++ b/frontend/apps/artcraft-website/src/pages/create-vfx/index.ts @@ -0,0 +1 @@ +export { default } from "./create-vfx"; diff --git a/frontend/apps/artcraft-website/tsconfig.app.json b/frontend/apps/artcraft-website/tsconfig.app.json index 37d25249a0..a710520fa6 100644 --- a/frontend/apps/artcraft-website/tsconfig.app.json +++ b/frontend/apps/artcraft-website/tsconfig.app.json @@ -93,6 +93,9 @@ }, { "path": "../../libs/api/tsconfig.lib.json" + }, + { + "path": "../../libs/components/vfx/tsconfig.lib.json" } ] } diff --git a/frontend/apps/artcraft-website/tsconfig.json b/frontend/apps/artcraft-website/tsconfig.json index 153ac3c55e..0554d6b598 100644 --- a/frontend/apps/artcraft-website/tsconfig.json +++ b/frontend/apps/artcraft-website/tsconfig.json @@ -5,6 +5,9 @@ { "path": "../../libs/components/slider-v2" }, + { + "path": "../../libs/components/vfx" + }, { "path": "../../libs/components/gallery-modal" }, diff --git a/frontend/apps/artcraft/app/src/components/signaled/TopBar/TopBar.tsx b/frontend/apps/artcraft/app/src/components/signaled/TopBar/TopBar.tsx index 62bd8378e5..c22219d4b2 100644 --- a/frontend/apps/artcraft/app/src/components/signaled/TopBar/TopBar.tsx +++ b/frontend/apps/artcraft/app/src/components/signaled/TopBar/TopBar.tsx @@ -576,6 +576,8 @@ export const TopBar = ({ pageName }: Props) => { return "Image to 3D World"; case "APPS": return "ArtCraft Apps"; + case "BACKGROUND_CHANGE": + return "Background Change"; default: return "Artcraft"; } diff --git a/frontend/apps/artcraft/app/src/config/appMenu.tsx b/frontend/apps/artcraft/app/src/config/appMenu.tsx index 02ec7ee46d..b935d5e51b 100644 --- a/frontend/apps/artcraft/app/src/config/appMenu.tsx +++ b/frontend/apps/artcraft/app/src/config/appMenu.tsx @@ -10,6 +10,7 @@ import { faWandMagicSparkles, faPenNib, faCrosshairs, + faSparkles, } from "@fortawesome/pro-solid-svg-icons"; import { useMemo } from "react"; import { @@ -31,7 +32,8 @@ export type AppId = | "IMAGE_TO_3D_WORLD" | "REMOVE_BACKGROUND" | "ANGLES" - | "STORYBOARD"; + | "STORYBOARD" + | "BACKGROUND_CHANGE"; export interface AppDescriptor { id: AppId; @@ -185,6 +187,16 @@ export const ALL_APPS: FullAppItem[] = [ color: "bg-fuchsia-600/40", badge: "NEW", }, + { + id: "background-change", + label: "Background Change", + description: "Swap the backdrop of a video using a reference image", + icon: faSparkles, + category: "edit", + action: "BACKGROUND_CHANGE", + color: "bg-orange-500/40", + badge: "NEW", + }, { id: "2d-canvas", label: "Image Editor", @@ -214,9 +226,12 @@ export const useVisibleApps = (): FullAppItem[] => { const storyboardEnabled = useStoryboardPageEnabled(); return useMemo( () => - ALL_APPS.filter((app) => - app.action === "STORYBOARD" ? storyboardEnabled : true, - ), + ALL_APPS.filter((app) => { + // Background Change is hidden in the desktop app for now. + if (app.action === "BACKGROUND_CHANGE") return false; + if (app.action === "STORYBOARD") return storyboardEnabled; + return true; + }), [storyboardEnabled], ); }; @@ -266,6 +281,7 @@ export const goToApp = (action?: string) => { "REMOVE_BACKGROUND", "ANGLES", "STORYBOARD", + "BACKGROUND_CHANGE", ].includes(action) ) { if (action === "STORYBOARD") { diff --git a/frontend/apps/artcraft/app/src/pages/MainApp.tsx b/frontend/apps/artcraft/app/src/pages/MainApp.tsx index a6ba3eb404..c116a91a35 100644 --- a/frontend/apps/artcraft/app/src/pages/MainApp.tsx +++ b/frontend/apps/artcraft/app/src/pages/MainApp.tsx @@ -71,6 +71,7 @@ import { ImageTo3DWorld } from "./PageImageTo3DWorld"; import { RemoveBackground } from "./PageRemoveBackground"; import { Angles } from "./PageAngles"; import { Storyboard } from "./PageStoryboard"; +import { PageBackgroundChange } from "./PageBackgroundChange"; import { PageScene } from "./PageScene"; import { topNavMediaId, @@ -251,6 +252,8 @@ const TabBody = ({ sceneToken }: { sceneToken?: string }) => { return
; case "STORYBOARD": return storyboardPageEnabled ?
: null; + case "BACKGROUND_CHANGE": + return
; default: return null; } diff --git a/frontend/apps/artcraft/app/src/pages/PageBackgroundChange/PageBackgroundChange.tsx b/frontend/apps/artcraft/app/src/pages/PageBackgroundChange/PageBackgroundChange.tsx new file mode 100644 index 0000000000..5e08160f32 --- /dev/null +++ b/frontend/apps/artcraft/app/src/pages/PageBackgroundChange/PageBackgroundChange.tsx @@ -0,0 +1,203 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faSparkles } from "@fortawesome/pro-solid-svg-icons"; +import { toast } from "@storyteller/ui-toaster"; +import { UploadImageMedia, UploadVideoMedia } from "@storyteller/api"; +import { + PromptBoxVFX, + SubTabStrip, + TruchetPattern, + VFXResultCard, + VFXShowcaseView, + newIdempotencyToken, + submitVFXJob, + useVFXStore, + VFX_NOT_AVAILABLE_ERROR, + VFX_SHOWCASE, +} from "@storyteller/ui-vfx"; + +export const PageBackgroundChange = () => { + const subTab = useVFXStore((s) => s.subTab); + const setSubTab = useVFXStore((s) => s.setSubTab); + const history = useVFXStore((s) => s.history); + const startResult = useVFXStore((s) => s.startResult); + const failResult = useVFXStore((s) => s.failResult); + const dismissResult = useVFXStore((s) => s.dismissResult); + const loadFromShowcase = useVFXStore((s) => s.loadFromShowcase); + const selectedShowcaseId = useVFXStore((s) => s.selectedShowcaseId); + const source = useVFXStore((s) => s.source); + const mask = useVFXStore((s) => s.mask); + const reference = useVFXStore((s) => s.reference); + const prompt = useVFXStore((s) => s.prompt); + const resolution = useVFXStore((s) => s.resolution); + + const promptBoxRef = useRef(null); + const [promptBoxHeight, setPromptBoxHeight] = useState(96); + const [isSubmitting, setIsSubmitting] = useState(false); + + useEffect(() => { + const el = promptBoxRef.current; + if (!el || typeof ResizeObserver === "undefined") return; + const update = () => setPromptBoxHeight(el.offsetHeight); + update(); + const ro = new ResizeObserver(update); + ro.observe(el); + return () => ro.disconnect(); + }, []); + + const handleSubmit = useCallback(async () => { + if (!source || !reference || isSubmitting) return; + setIsSubmitting(true); + const id = startResult(); + const trimmed = prompt.trim(); + const result = await submitVFXJob({ + source_video_media_token: source.mediaToken, + reference_image_media_token: reference.mediaToken, + prompt: trimmed.length > 0 ? trimmed : null, + uuid_idempotency_token: newIdempotencyToken(), + }); + setIsSubmitting(false); + + if (!result.success) { + const isExpected = result.error_code_str === VFX_NOT_AVAILABLE_ERROR; + const message = isExpected + ? "Background change backend coming soon. Your inputs are saved." + : result.error_message || "Failed to submit background change job."; + failResult(id, message); + if (isExpected) toast(message); + else toast.error(message); + } + }, [source, reference, prompt, isSubmitting, startResult, failResult]); + + const handleTryShowcase = useCallback(() => { + const entry = VFX_SHOWCASE.find((e) => e.id === selectedShowcaseId); + if (!entry) return; + loadFromShowcase({ + prompt: entry.prompt, + resolution: entry.resolution, + source: { + id: entry.source.mediaToken, + url: entry.source.url, + mediaToken: entry.source.mediaToken, + }, + mask: entry.mask + ? { + id: entry.mask.mediaToken, + url: entry.mask.url, + mediaToken: entry.mask.mediaToken, + } + : undefined, + reference: entry.reference + ? { + id: entry.reference.mediaToken, + url: entry.reference.url, + mediaToken: entry.reference.mediaToken, + } + : undefined, + }); + toast("Loaded showcase. Edit and Generate."); + }, [selectedShowcaseId, loadFromShowcase]); + + return ( +
+ + + {subTab === "showcase" ? ( +
+ +
+ ) : history.length === 0 ? ( +
+
+ +
+
+ +
+
+ ) : ( +
+
+ {history.map((r) => ( + dismissResult(r.id)} + /> + ))} +
+
+ )} + +
+ +
+
+ toast.error(msg)} + /> +
+
+
+ ); +}; + +interface EmptyStateProps { + title: string; + subtitle: string; +} + +const EmptyState = ({ title, subtitle }: EmptyStateProps) => ( +
+
+ +
+

{title}

+

{subtitle}

+
+); + +export default PageBackgroundChange; diff --git a/frontend/apps/artcraft/app/src/pages/PageBackgroundChange/index.ts b/frontend/apps/artcraft/app/src/pages/PageBackgroundChange/index.ts new file mode 100644 index 0000000000..6530419c04 --- /dev/null +++ b/frontend/apps/artcraft/app/src/pages/PageBackgroundChange/index.ts @@ -0,0 +1 @@ +export { PageBackgroundChange } from "./PageBackgroundChange"; diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx b/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx index f180afad44..431c2c6270 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from "react"; +import { useMemo } from "react"; import { PageDraw as PageDrawLib, useSceneStore, @@ -97,7 +97,7 @@ const useTauriAdapter = (): PageDrawAdapter => { enqueueEditImage: async (req) => { const request: GenerateImageRequest = { model: req.model, - scene_image_media_token: req.sceneImageMediaToken, + canvas_image_media_token: req.canvasImageMediaToken, image_media_tokens: req.imageMediaTokens, prompt: req.prompt, enable_system_prompt: diff --git a/frontend/apps/artcraft/app/src/pages/Stores/TabState.ts b/frontend/apps/artcraft/app/src/pages/Stores/TabState.ts index c23b50dbbd..4291562f4e 100644 --- a/frontend/apps/artcraft/app/src/pages/Stores/TabState.ts +++ b/frontend/apps/artcraft/app/src/pages/Stores/TabState.ts @@ -15,7 +15,8 @@ export type TabId = | "IMAGE_TO_3D_WORLD" | "REMOVE_BACKGROUND" | "ANGLES" - | "STORYBOARD"; + | "STORYBOARD" + | "BACKGROUND_CHANGE"; const DEFAULT_TAB: TabId = "IMAGE"; diff --git a/frontend/apps/artcraft/tsconfig.json b/frontend/apps/artcraft/tsconfig.json index e1b109a5be..f61ca26ae3 100644 --- a/frontend/apps/artcraft/tsconfig.json +++ b/frontend/apps/artcraft/tsconfig.json @@ -55,6 +55,9 @@ { "path": "../../libs/components/promptbox" }, + { + "path": "../../libs/components/vfx" + }, { "path": "../../libs/components/menu-icon-selector" }, diff --git a/frontend/libs/api/src/lib/MediaUploadApi.ts b/frontend/libs/api/src/lib/MediaUploadApi.ts index 3685e6e000..bae21c1e14 100644 --- a/frontend/libs/api/src/lib/MediaUploadApi.ts +++ b/frontend/libs/api/src/lib/MediaUploadApi.ts @@ -4,10 +4,15 @@ import { FilterEngineCategories } from "./enums/QueryFilters.js"; import { Visibility } from "./enums/Visibility.js"; import { EIntermediateFile } from "./enums/EIntermediateFile.js"; import { UploaderStates, UploaderState } from "@storyteller/common"; + +type GetSessionTokenForUploadStuioShotResult = + | { success: true; data?: string } + | { success: false; errorMessage?: any }; + export class MediaUploadApi extends ApiManager { private sessionToken = ""; - private async getSessionTokenForUploadStudioShot() { + private async getSessionTokenForUploadStudioShot(): Promise { const endpoint = `${this.getApiSchemeAndHost()}/v1/session_token`; return await this.get<{ maybe_signed_session: string; diff --git a/frontend/libs/api/src/lib/PromptsApi.ts b/frontend/libs/api/src/lib/PromptsApi.ts index d46be69881..b093ec3e33 100644 --- a/frontend/libs/api/src/lib/PromptsApi.ts +++ b/frontend/libs/api/src/lib/PromptsApi.ts @@ -106,7 +106,6 @@ export class PromptsApi extends ApiManager { console.log(response); const success = response.success ?? false; - const status = response.status ?? ""; const errorMessage = response.BadInput; return { diff --git a/frontend/libs/api/src/lib/config/StorytellerApiHostStore.ts b/frontend/libs/api/src/lib/config/StorytellerApiHostStore.ts index fe62be5e65..c51cd74126 100644 --- a/frontend/libs/api/src/lib/config/StorytellerApiHostStore.ts +++ b/frontend/libs/api/src/lib/config/StorytellerApiHostStore.ts @@ -39,6 +39,12 @@ export class StorytellerApiHostStore { return instance; } + /** Convert an API path to a full URL using the configured scheme and host. */ + public pathToFullUrl(path: string): string { + const trimmedPath = path.startsWith("/") ? path.slice(1) : path; + return `${this.apiSchemeAndHost}/${trimmedPath}`; + } + /** Get the API scheme and host. */ public getApiSchemeAndHost(): string { console.debug("StorytellerApiHostStore.getApiSchemeAndHost()", this.apiSchemeAndHost, this.constructor.name); diff --git a/frontend/libs/api/src/lib/models/Users.ts b/frontend/libs/api/src/lib/models/Users.ts index 5f7fe39d47..8fa2cf9560 100644 --- a/frontend/libs/api/src/lib/models/Users.ts +++ b/frontend/libs/api/src/lib/models/Users.ts @@ -1,4 +1,4 @@ -import { USER_FEATURE_FLAGS } from "../../libs/api/src/lib/enums/UserFeatures"; +import { USER_FEATURE_FLAGS } from "../enums/UserFeatures"; export interface CoreInfo { default_avatar: { diff --git a/frontend/libs/components/button/src/lib/GenerateButton.tsx b/frontend/libs/components/button/src/lib/GenerateButton.tsx index ee70d2392d..896f4a7bdf 100644 --- a/frontend/libs/components/button/src/lib/GenerateButton.tsx +++ b/frontend/libs/components/button/src/lib/GenerateButton.tsx @@ -17,7 +17,7 @@ export const GenerateButton = ({ return (
@@ -1005,7 +1182,8 @@ const PageDraw = ({ adapter }: PageDrawProps) => { EncodeImageBitmapToBase64={EncodeImageBitmapToBase64} onGenerateClick={handleGenerate} onFitPressed={onFitPressed} - isDisabled={false} + isDisabled={!baseImageBitmap && !baseImageInfo?.isBlankCanvas} + isEnqueueing={isGenerating} generationCount={generationCount} onGenerationCountChange={setGenerationCount} selectedImageModel={selectedImageModel} diff --git a/frontend/libs/components/pagedraw/src/lib/adapter.ts b/frontend/libs/components/pagedraw/src/lib/adapter.ts index 7a4dd61896..c2931de485 100644 --- a/frontend/libs/components/pagedraw/src/lib/adapter.ts +++ b/frontend/libs/components/pagedraw/src/lib/adapter.ts @@ -7,7 +7,7 @@ import type { BaseSelectorImage } from "./types"; export interface PageDrawEditRequest { model?: ImageModel; provider?: GenerationProvider; - sceneImageMediaToken?: string; + canvasImageMediaToken?: string; imageMediaTokens?: string[]; prompt?: string; disableSystemPrompt?: boolean; diff --git a/frontend/libs/components/pagedraw/src/lib/stores/SceneState.ts b/frontend/libs/components/pagedraw/src/lib/stores/SceneState.ts index 1012e92f53..43ecf2d4bb 100644 --- a/frontend/libs/components/pagedraw/src/lib/stores/SceneState.ts +++ b/frontend/libs/components/pagedraw/src/lib/stores/SceneState.ts @@ -3,6 +3,14 @@ import { Node, NodeType } from "../Node"; import { ImageBundle, BaseSelectorImage } from "../types"; import { Model3DParams } from "../utilities/render3DModel"; +// Generation counter for in-flight base-image loads. Incremented on every +// setBaseImageInfo call so that a stale `.onload` from a superseded +// selection (e.g. rapid HistoryStack navigation while the previous image is +// still in flight) becomes a no-op. Without this guard, A's onload could fire +// after B was selected and overwrite baseImageBitmap (and via applyImageNodes, +// even drawNodes) with A's content. +let baseImageLoadGen = 0; + // LineNode type — ordering in PageDraw is by position in drawNodes array (zIndex ignored there) export type LineNode = { id: string; @@ -1372,8 +1380,11 @@ export const useSceneStore = create((set, get, store) => ({ }, setBaseImageInfo: (image: BaseSelectorImage | null) => { + // Bump first so any in-flight load from a previous call invalidates itself + // on completion (see imgBitmap.onload / onerror below). + const myGen = ++baseImageLoadGen; if (!image) { - set({ baseImageInfo: null }); + set({ baseImageInfo: null, baseImageBitmap: null }); return; } @@ -1438,13 +1449,22 @@ export const useSceneStore = create((set, get, store) => ({ return; } - set({ baseImageInfo: image }); + // Clear baseImageBitmap synchronously so consumers can distinguish + // "loaded for current info" from "stale from previous info" purely by + // checking truthiness. Without this, baseImageInfo points at the new + // image while baseImageBitmap still holds the previous one, and any code + // gating on bitmap presence (e.g. PageDraw's Generate button) would let + // through a click that snapshots the previous pixels with the new + // image's mediaToken. applyImageNodes will repopulate it on onload. + set({ baseImageInfo: image, baseImageBitmap: null }); const imgBitmap = new Image(); imgBitmap.onload = () => { + if (myGen !== baseImageLoadGen) return; // superseded by a newer setBaseImageInfo applyImageNodes(imgBitmap); }; imgBitmap.onerror = (event) => { + if (myGen !== baseImageLoadGen) return; console.error("Failed to load base image, discarding", event); set({ baseImageInfo: null, baseImageBitmap: null }); imgBitmap.onload = null; diff --git a/frontend/libs/components/pagedraw/src/lib/utilities/generatePipeline.ts b/frontend/libs/components/pagedraw/src/lib/utilities/generatePipeline.ts new file mode 100644 index 0000000000..12848ecda9 --- /dev/null +++ b/frontend/libs/components/pagedraw/src/lib/utilities/generatePipeline.ts @@ -0,0 +1,117 @@ +import GeneratePipelineWorker from "./generatePipeline.worker.ts?worker&inline"; + +type CompositeOk = { id: number; ok: true; type: "composite"; blob: Blob }; +type MaskOk = { id: number; ok: true; type: "mask"; bytes: Uint8Array }; +type WorkerErr = { id: number; ok: false; error: string }; +type WorkerResponse = CompositeOk | MaskOk | WorkerErr; + +type Pending = { + resolve: (value: CompositeOk | MaskOk) => void; + reject: (err: Error) => void; +}; + +let workerInstance: Worker | null = null; +let nextId = 1; +const pendingRequests = new Map(); + +const ensureWorker = (): Worker => { + if (workerInstance) return workerInstance; + const worker = new GeneratePipelineWorker(); + worker.addEventListener("message", (event: MessageEvent) => { + const res = event.data; + const pending = pendingRequests.get(res.id); + if (!pending) return; + pendingRequests.delete(res.id); + if (res.ok) { + pending.resolve(res); + } else { + pending.reject(new Error(res.error)); + } + }); + worker.addEventListener("error", (event) => { + const err = new Error(event.message || "Generate pipeline worker error"); + for (const pending of pendingRequests.values()) { + pending.reject(err); + } + pendingRequests.clear(); + }); + workerInstance = worker; + return worker; +}; + +// postMessage can throw synchronously (data not structured-cloneable, a +// transferable already detached, etc). Without this, the pendingRequests +// entry would stay registered forever and the returned Promise would hang. +const postOrReject = ( + worker: Worker, + id: number, + message: unknown, + transfer: Transferable[], + reject: (err: Error) => void, +): void => { + try { + worker.postMessage(message, transfer); + } catch (err) { + pendingRequests.delete(id); + reject(err instanceof Error ? err : new Error(String(err))); + } +}; + +export const compositeInWorker = (params: { + markerBitmap: ImageBitmap; + baseBitmap?: ImageBitmap; + width: number; + height: number; +}): Promise => { + const worker = ensureWorker(); + const id = nextId++; + const transfer: Transferable[] = [params.markerBitmap]; + if (params.baseBitmap) transfer.push(params.baseBitmap); + return new Promise((resolve, reject) => { + pendingRequests.set(id, { + resolve: (res) => { + if (res.type !== "composite") { + reject(new Error("Unexpected response type for composite")); + return; + } + resolve(res.blob); + }, + reject, + }); + postOrReject( + worker, + id, + { id, type: "composite", ...params }, + transfer, + reject, + ); + }); +}; + +export const maskInWorker = (params: { + markerBitmap: ImageBitmap; + width: number; + height: number; +}): Promise => { + const worker = ensureWorker(); + const id = nextId++; + return new Promise((resolve, reject) => { + pendingRequests.set(id, { + resolve: (res) => { + if (res.type !== "mask") { + reject(new Error("Unexpected response type for mask")); + return; + } + resolve(res.bytes); + }, + reject, + }); + postOrReject( + worker, + id, + { id, type: "mask", ...params }, + [params.markerBitmap], + reject, + ); + }); +}; diff --git a/frontend/libs/components/pagedraw/src/lib/utilities/generatePipeline.worker.ts b/frontend/libs/components/pagedraw/src/lib/utilities/generatePipeline.worker.ts new file mode 100644 index 0000000000..79976c4bb8 --- /dev/null +++ b/frontend/libs/components/pagedraw/src/lib/utilities/generatePipeline.worker.ts @@ -0,0 +1,81 @@ +/// + +type WorkerRequest = + | { + id: number; + type: "composite"; + markerBitmap: ImageBitmap; + baseBitmap?: ImageBitmap; + width: number; + height: number; + } + | { + id: number; + type: "mask"; + markerBitmap: ImageBitmap; + width: number; + height: number; + }; + +type WorkerResponse = + | { id: number; ok: true; type: "composite"; blob: Blob } + | { id: number; ok: true; type: "mask"; bytes: Uint8Array } + | { id: number; ok: false; error: string }; + +const ctxOf = (canvas: OffscreenCanvas): OffscreenCanvasRenderingContext2D => { + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Failed to get OffscreenCanvas 2d context"); + return ctx; +}; + +const post = (response: WorkerResponse, transfer: Transferable[] = []) => { + (self as unknown as DedicatedWorkerGlobalScope).postMessage( + response, + transfer, + ); +}; + +self.addEventListener("message", async (event: MessageEvent) => { + const req = event.data; + try { + if (req.type === "composite") { + const canvas = new OffscreenCanvas(req.width, req.height); + const ctx = ctxOf(canvas); + if (req.baseBitmap) { + ctx.drawImage(req.baseBitmap, 0, 0, req.width, req.height); + } else { + ctx.fillStyle = "white"; + ctx.fillRect(0, 0, req.width, req.height); + } + ctx.drawImage(req.markerBitmap, 0, 0, req.width, req.height); + const blob = await canvas.convertToBlob({ type: "image/png" }); + post({ id: req.id, ok: true, type: "composite", blob }); + } else if (req.type === "mask") { + const canvas = new OffscreenCanvas(req.width, req.height); + const ctx = ctxOf(canvas); + ctx.drawImage(req.markerBitmap, 0, 0, req.width, req.height); + const blob = await canvas.convertToBlob({ type: "image/png" }); + const buffer = await blob.arrayBuffer(); + const bytes = new Uint8Array(buffer); + post({ id: req.id, ok: true, type: "mask", bytes }, [bytes.buffer]); + } + } catch (err) { + post({ + id: req.id, + ok: false, + error: err instanceof Error ? err.message : String(err), + }); + } finally { + // Always release transferred ImageBitmaps regardless of success/failure + // path. Without this, a throw mid-pipeline (e.g. an OOM during drawImage + // on a 4K canvas) would leak ~67MB of bitmap data per failed bake. + if (req.type === "composite") { + req.baseBitmap?.close(); + req.markerBitmap.close(); + } else if (req.type === "mask") { + req.markerBitmap.close(); + } + } +}); + +export {}; diff --git a/frontend/libs/components/popover/src/lib/popover.tsx b/frontend/libs/components/popover/src/lib/popover.tsx index 42adb8c6fe..a694b8253d 100644 --- a/frontend/libs/components/popover/src/lib/popover.tsx +++ b/frontend/libs/components/popover/src/lib/popover.tsx @@ -320,7 +320,7 @@ export const PopoverMenu = ({ }; const className = twMerge( - "text-sm font-medium rounded-lg px-3 py-2 shadow-sm", + "text-sm font-medium rounded-lg px-2.5 py-1.5 shadow-sm", "flex gap-2 items-center justify-center outline-none", "transition-all duration-150", "bg-ui-controls px-3 text-base-fg hover:bg-ui-controls/80 border border-ui-controls-border", @@ -609,10 +609,10 @@ export const PopoverMenu = ({ style={ maxListHeight ? { - maxHeight: maxListHeight, - overflowY: "auto", - opacity: scrollReady ? 1 : 0, - } + maxHeight: maxListHeight, + overflowY: "auto", + opacity: scrollReady ? 1 : 0, + } : undefined } > @@ -620,10 +620,10 @@ export const PopoverMenu = ({ const tooltipContent = typeof item.hoverTooltip === "function" ? ( - item.hoverTooltip as ( - close: () => void, - ) => ReactNode - )(close) + item.hoverTooltip as ( + close: () => void, + ) => ReactNode + )(close) : item.hoverTooltip; const itemRow = ( @@ -789,10 +789,10 @@ export const PopoverMenu = ({ content={ typeof item.hoverTooltip === "function" ? ( - item.hoverTooltip as ( - close: () => void, - ) => ReactNode - )(close) + item.hoverTooltip as ( + close: () => void, + ) => ReactNode + )(close) : item.hoverTooltip } position="right" diff --git a/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx b/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx index 0580e59688..e12e9861f3 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx @@ -509,7 +509,7 @@ export const PromptBox2D = ({ className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50" icon={undefined} onClick={handleGenerate} - disabled={isEnqueueing || !prompt.trim()} + disabled={isEnqueueing || isDisabled || !prompt.trim()} loading={isEnqueueing} > Generate diff --git a/frontend/libs/components/promptbox/src/lib/common/GenerationCountPicker.tsx b/frontend/libs/components/promptbox/src/lib/common/GenerationCountPicker.tsx index 300d141ba8..85d501340a 100644 --- a/frontend/libs/components/promptbox/src/lib/common/GenerationCountPicker.tsx +++ b/frontend/libs/components/promptbox/src/lib/common/GenerationCountPicker.tsx @@ -61,7 +61,6 @@ export const GenerationCountPicker = ({ mode="toggle" panelTitle="No. of images" triggerIcon={} - buttonClassName="h-9" /> ); diff --git a/frontend/libs/components/vfx/package.json b/frontend/libs/components/vfx/package.json new file mode 100644 index 0000000000..74120a4aae --- /dev/null +++ b/frontend/libs/components/vfx/package.json @@ -0,0 +1,17 @@ +{ + "name": "@storyteller/ui-vfx", + "version": "0.0.1", + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "development": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + } +} diff --git a/frontend/libs/components/vfx/src/index.ts b/frontend/libs/components/vfx/src/index.ts new file mode 100644 index 0000000000..448eee8704 --- /dev/null +++ b/frontend/libs/components/vfx/src/index.ts @@ -0,0 +1,10 @@ +export * from "./lib/types"; +export * from "./lib/store"; +export * from "./lib/api"; +export * from "./lib/showcase-fixtures"; +export * from "./lib/SubTabStrip"; +export * from "./lib/ShowcaseCarousel"; +export * from "./lib/VFXResultCard"; +export * from "./lib/VFXShowcaseView"; +export * from "./lib/PromptBoxVFX"; +export * from "./lib/TruchetPattern"; diff --git a/frontend/libs/components/vfx/src/lib/PromptBoxVFX.tsx b/frontend/libs/components/vfx/src/lib/PromptBoxVFX.tsx new file mode 100644 index 0000000000..faee807deb --- /dev/null +++ b/frontend/libs/components/vfx/src/lib/PromptBoxVFX.tsx @@ -0,0 +1,566 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { twMerge } from "tailwind-merge"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import type { IconDefinition } from "@fortawesome/fontawesome-svg-core"; +import { + faSpinnerThird, + faVideo, + faImage, + faXmark, + faSparkles, + faPlus, + faImages, + faPenToSquare, +} from "@fortawesome/pro-solid-svg-icons"; +import { Button, GenerateButton } from "@storyteller/ui-button"; +import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover"; +import { Tooltip } from "@storyteller/ui-tooltip"; +import { GalleryModal, type GalleryItem } from "@storyteller/ui-gallery-modal"; +import { UploaderStates, type UploaderState } from "@storyteller/common"; +import { useVFXStore } from "./store"; +import { + MAX_SOURCE_DURATION_S, + RESOLUTION_OPTIONS, + VFX_MODELS, + type VFXModelId, + type VFXResolution, +} from "./types"; + +export type VFXUploadFn = (args: { + title: string; + assetFile: File; + progressCallback: (newState: UploaderState) => void; +}) => Promise; + +interface PromptBoxVFXProps { + onSubmit: () => void; + isSubmitting: boolean; + uploadVideo: VFXUploadFn; + uploadImage: VFXUploadFn; + onError: (message: string) => void; + containerClassName?: string; + hideResolution?: boolean; +} + +type SlotKind = "source" | "mask" | "reference"; + +export const PromptBoxVFX = ({ + onSubmit, + isSubmitting, + uploadVideo, + uploadImage, + onError, + containerClassName, + hideResolution = false, +}: PromptBoxVFXProps) => { + const source = useVFXStore((s) => s.source); + const reference = useVFXStore((s) => s.reference); + const prompt = useVFXStore((s) => s.prompt); + const resolution = useVFXStore((s) => s.resolution); + const selectedModelId = useVFXStore((s) => s.selectedModelId); + const setSource = useVFXStore((s) => s.setSource); + const setMask = useVFXStore((s) => s.setMask); + const setReference = useVFXStore((s) => s.setReference); + const setPrompt = useVFXStore((s) => s.setPrompt); + const setResolution = useVFXStore((s) => s.setResolution); + const setSelectedModelId = useVFXStore((s) => s.setSelectedModelId); + const selectedModel = + VFX_MODELS.find((m) => m.id === selectedModelId) ?? VFX_MODELS[0]; + + const sourceInputRef = useRef(null); + const maskInputRef = useRef(null); + const referenceInputRef = useRef(null); + const promptTextareaRef = useRef(null); + const [sourceUploading, setSourceUploading] = useState(false); + const [maskUploading, setMaskUploading] = useState(false); + const [referenceUploading, setReferenceUploading] = useState(false); + const [showPromptPopover, setShowPromptPopover] = useState(false); + + useEffect(() => { + if (showPromptPopover) { + requestAnimationFrame(() => promptTextareaRef.current?.focus()); + } + }, [showPromptPopover]); + + const [galleryOpen, setGalleryOpen] = useState(false); + const [galleryTarget, setGalleryTarget] = useState("source"); + const [gallerySelected, setGallerySelected] = useState([]); + + const handleSourceUpload = useCallback( + async (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + if (sourceInputRef.current) sourceInputRef.current.value = ""; + if (!file) return; + + const previewUrl = URL.createObjectURL(file); + const duration = await measureVideoDuration(previewUrl); + if (duration > MAX_SOURCE_DURATION_S) { + onError(`Source video must be ${MAX_SOURCE_DURATION_S}s or shorter`); + URL.revokeObjectURL(previewUrl); + return; + } + + setSourceUploading(true); + await uploadVideo({ + title: `vfx-source-${Math.random().toString(36).slice(2, 10)}`, + assetFile: file, + progressCallback: (state) => { + if (state.status === UploaderStates.success && state.data) { + setSource({ + id: Math.random().toString(36).slice(2), + url: previewUrl, + mediaToken: state.data, + }); + setSourceUploading(false); + } else if ( + state.status === UploaderStates.assetError || + state.status === UploaderStates.imageCreateError + ) { + onError("Failed to upload source video. Use an MP4 file."); + setSourceUploading(false); + URL.revokeObjectURL(previewUrl); + } + }, + }); + }, + [setSource, uploadVideo, onError], + ); + + const handleImageUpload = useCallback( + async ( + event: React.ChangeEvent, + kind: "mask" | "reference", + ) => { + const file = event.target.files?.[0]; + const inputRef = kind === "mask" ? maskInputRef : referenceInputRef; + if (inputRef.current) inputRef.current.value = ""; + if (!file) return; + + const previewUrl = URL.createObjectURL(file); + const setUploading = + kind === "mask" ? setMaskUploading : setReferenceUploading; + const setRef = kind === "mask" ? setMask : setReference; + + setUploading(true); + await uploadImage({ + title: `vfx-${kind}-${Math.random().toString(36).slice(2, 10)}`, + assetFile: file, + progressCallback: (state) => { + if (state.status === UploaderStates.success && state.data) { + setRef({ + id: Math.random().toString(36).slice(2), + url: previewUrl, + mediaToken: state.data, + }); + setUploading(false); + } else if ( + state.status === UploaderStates.assetError || + state.status === UploaderStates.imageCreateError + ) { + onError( + kind === "mask" + ? "Failed to upload mask image." + : "Failed to upload reference image.", + ); + setUploading(false); + URL.revokeObjectURL(previewUrl); + } + }, + }); + }, + [setMask, setReference, uploadImage, onError], + ); + + const openGallery = useCallback((target: SlotKind) => { + setGalleryTarget(target); + setGallerySelected([]); + setGalleryOpen(true); + }, []); + + const handleGallerySelectItem = useCallback((id: string) => { + // Single-select: replace. + setGallerySelected([id]); + }, []); + + const handleGalleryUseSelected = useCallback( + (items: GalleryItem[]) => { + const item = items[0]; + if (!item) { + setGalleryOpen(false); + return; + } + const ref = { + id: Math.random().toString(36).slice(2), + url: item.fullImage || item.thumbnail || "", + mediaToken: item.id, + }; + if (galleryTarget === "source") setSource(ref); + else if (galleryTarget === "mask") setMask(ref); + else setReference(ref); + setGalleryOpen(false); + setGallerySelected([]); + }, + [galleryTarget, setSource, setMask, setReference], + ); + + const triggerUpload = useCallback((target: SlotKind) => { + if (target === "source") sourceInputRef.current?.click(); + else if (target === "mask") maskInputRef.current?.click(); + else referenceInputRef.current?.click(); + }, []); + + const resolutionItems: PopoverItem[] = RESOLUTION_OPTIONS.map((r) => ({ + label: r, + selected: r === resolution, + action: r, + })); + + const handleResolutionSelect = useCallback( + (item: PopoverItem) => { + const next = (item.action ?? item.label) as VFXResolution; + if (RESOLUTION_OPTIONS.includes(next)) setResolution(next); + }, + [setResolution], + ); + + const modelItems: PopoverItem[] = VFX_MODELS.map((m) => ({ + label: m.label, + description: m.description, + selected: m.id === selectedModelId, + action: m.id, + })); + + const handleModelSelect = useCallback( + (item: PopoverItem) => { + const next = (item.action ?? "") as VFXModelId; + if (VFX_MODELS.some((m) => m.id === next)) setSelectedModelId(next); + }, + [setSelectedModelId], + ); + + const canSubmit = + !!source && + !!reference && + !isSubmitting && + !sourceUploading && + !maskUploading && + !referenceUploading; + + const hasPrompt = prompt.trim().length > 0; + + return ( +
+ {showPromptPopover && ( +
+
+
+ + Prompt (optional) + + +
+