diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs index f028dc7de4..854a26fffb 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs @@ -1,5 +1,5 @@ -use crate::core::commands::enqueue::image::handle_fal::handle_fal; -use crate::core::commands::enqueue::image::handle_sora::handle_sora; +use crate::core::commands::enqueue::image::handle_image_fal::handle_image_fal; +use crate::core::commands::enqueue::image::handle_image_sora::handle_image_sora; use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::commands::response::shorthand::Response; @@ -24,7 +24,7 @@ use base64::prelude::BASE64_STANDARD; use base64::{DecodeError, Engine}; use errors::{AnyhowError, AnyhowResult}; use fal_client::error::fal_error_plus::FalErrorPlus; -use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; use filesys::file_read_bytes::file_read_bytes; use image::codecs::png::{CompressionType, FilterType, PngEncoder}; @@ -55,6 +55,8 @@ use storyteller_client::media_files::upload_image_media_file_from_file::upload_i use storyteller_client::utils::api_host::ApiHost; use tauri::{AppHandle, Emitter, Manager, State}; use tempfile::NamedTempFile; +use crate::core::commands::enqueue::image::handle_image_artcraft::handle_image_artcraft; +use crate::core::commands::enqueue::object::handle_object_artcraft::handle_object_artcraft; #[derive(Deserialize)] pub struct EnqueueTextToImageRequest { @@ -93,6 +95,8 @@ pub enum EnqueueTextToImageErrorType { NeedsFalApiKey, /// Fal had an API error FalError, + /// Needs to be logged into Artcraft + NeedsStorytellerCredentials, } @@ -172,13 +176,20 @@ pub async fn handle_request( None => { return Err(InternalImageError::NoModelSpecified); } - Some(EnqueueTextToImageModel::FluxProUltra) | Some(EnqueueTextToImageModel::Recraft3) => { - handle_fal(&app, request, fal_creds_manager, fal_task_queue).await?; - }, Some(EnqueueTextToImageModel::GptImage1) => { - handle_sora(&app, request, sora_creds_manager, sora_task_queue).await?; + handle_image_sora(&app, request, sora_creds_manager, sora_task_queue).await?; + return Ok(()); + } + _ => { + // Fall-through } }; + if fal_creds_manager.has_apparent_api_token()? { + handle_image_fal(&app, request, fal_creds_manager, fal_task_queue).await?; + } else { + handle_image_artcraft(request, &app, app_data_root, storyteller_creds_manager).await?; + } + Ok(()) } diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_artcraft.rs new file mode 100644 index 0000000000..3594e5117c --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_artcraft.rs @@ -0,0 +1,95 @@ +use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageModel, EnqueueTextToImageRequest}; +use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use artcraft_api_defs::generate::object::generate_hunyuan_2_image_to_3d::GenerateHunyuan2ImageTo3dRequest; +use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::GenerateKling16ProImageToVideoRequest; +use fal_client::creds::fal_api_key::FalApiKey; +use fal_client::requests::queue::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; +use idempotency::uuid::generate_random_uuid; +use log::{error, info, warn}; +use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image; +use storyteller_client::generate::object::generate_hunyuan2_image_to_3d::generate_hunyuan2_image_to_3d; +use storyteller_client::generate::video::generate_kling_16_pro_image_to_video::generate_kling_16_pro_image_to_video; +use storyteller_client::utils::api_host::ApiHost; +use tauri::{AppHandle, State}; + +pub async fn handle_image_artcraft( + request: EnqueueTextToImageRequest, + app: &AppHandle, + app_data_root: &AppDataRoot, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result<(), InternalImageError> { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + error!("No Artcraft credentials are set. Can't perform action."); + let event = + GenerationEnqueueFailureEvent::no_artcraft_credentials(GenerationAction::GenerateImage); + + if let Err(err) = event.send(&app) { + error!("Failed to emit event: {:?}", err); // Fail open. + } + + return Err(InternalImageError::NeedsStorytellerCredentials); + }, + }; + + info!("Calling Artcraft text to image..."); + + let uuid_idempotency_token = generate_random_uuid(); + + let mut selected_model = None; + + let job_token = match request.model { + None => { + return Err(InternalImageError::NoModelSpecified); + } + Some(EnqueueTextToImageModel::GptImage1) => { + return Err(InternalImageError::AnyhowError(anyhow!("wrong logic: artcraft is handling sora images"))); + } + Some(EnqueueTextToImageModel::Recraft3) => { + return Err(InternalImageError::AnyhowError(anyhow!("not yet implemented in Artcraft"))); + } + Some(EnqueueTextToImageModel::FluxProUltra) => { + info!("enqueue Flux Pro Ultra"); + selected_model = Some(GenerationModel::FluxPro11Ultra); + let request = GenerateFluxPro11UltraTextToImageRequest { + uuid_idempotency_token, + prompt: request.prompt, + }; + let result = generate_flux_pro_11_ultra_text_to_image( + &ApiHost::Storyteller, + Some(&creds), + request, + ).await; + match result { + Ok(enqueued) => { + info!("Successfully enqueued Artcraft flux pro ultra text to image generation"); + enqueued.inference_job_token + } + Err(err) => { + error!("Failed to use Artcraft flux pro ultra text to image generation: {:?}", err); + return Err(InternalImageError::StorytellerError(err)); + } + } + } + }; + + Ok(()) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_fal.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_fal.rs similarity index 93% rename from crates/desktop/artcraft/src/core/commands/enqueue/image/handle_fal.rs rename to crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_fal.rs index 1bf21d74f8..9f5e1d82c0 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_fal.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_fal.rs @@ -12,12 +12,12 @@ use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use anyhow::anyhow; use fal_client::creds::fal_api_key::FalApiKey; -use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; use log::{error, info, warn}; use tauri::AppHandle; -pub async fn handle_fal( +pub async fn handle_image_fal( app: &AppHandle, request: EnqueueTextToImageRequest, fal_creds_manager: &FalCredentialManager, @@ -56,7 +56,7 @@ pub async fn handle_fal( } Some(EnqueueTextToImageModel::FluxProUltra) => { info!("enqueue Flux Pro Ultra text-to-image with prompt: {}", prompt); - enqueue_flux_pro_ultra_text_to_image(FluxProUltraTextToImageArgs { + enqueue_flux_pro_11_ultra_text_to_image(FluxPro11UltraTextToImageArgs { prompt, api_key: &api_key, }).await diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_sora.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_sora.rs similarity index 95% rename from crates/desktop/artcraft/src/core/commands/enqueue/image/handle_sora.rs rename to crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_sora.rs index b8c018beec..b9d0ab4f3f 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_sora.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/handle_image_sora.rs @@ -11,7 +11,7 @@ use crate::services::sora::state::sora_credential_manager::SoraCredentialManager use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use anyhow::anyhow; -use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; use log::{error, info, warn}; use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; @@ -25,7 +25,7 @@ use tauri::AppHandle; const SORA_SIMPLE_IMAGE_GEN_TIMEOUT : Duration = Duration::from_millis(1000 * 30); // 30 seconds -pub async fn handle_sora( +pub async fn handle_image_sora( app: &AppHandle, request: EnqueueTextToImageRequest, sora_creds_manager: &SoraCredentialManager, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/internal_image_error.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/internal_image_error.rs index fae3383af2..800a22cc25 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/internal_image_error.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/internal_image_error.rs @@ -1,20 +1,23 @@ +use crate::core::artcraft_error::ArtcraftError; use base64::DecodeError; use errors::AnyhowError; use fal_client::error::fal_error_plus::FalErrorPlus; use openai_sora_client::sora_error::SoraError; -use storyteller_client::error::api_error::ApiError; +use storyteller_client::error::storyteller_error::StorytellerError; #[derive(Debug)] pub enum InternalImageError { NoModelSpecified, NeedsFalApiKey, NeedsSoraCredentials, + NeedsStorytellerCredentials, FalError(FalErrorPlus), SoraError(SoraError), AnyhowError(AnyhowError), - StorytellerApiError(ApiError), + StorytellerError(StorytellerError), DecodeError(DecodeError), IoError(std::io::Error), + ArtcraftError(ArtcraftError), } impl From for InternalImageError { @@ -35,9 +38,9 @@ impl From for InternalImageError { } } -impl From for InternalImageError { - fn from(value: ApiError) -> Self { - Self::StorytellerApiError(value) +impl From for InternalImageError { + fn from(value: StorytellerError) -> Self { + Self::StorytellerError(value) } } @@ -52,3 +55,15 @@ impl From for InternalImageError { Self::IoError(value) } } + +impl From for InternalImageError { + fn from(value: ArtcraftError) -> Self { + match value { + ArtcraftError::AnyhowError(e) => Self::AnyhowError(e), + ArtcraftError::DecodeError(e) => Self::DecodeError(e), + ArtcraftError::IoError(e) => Self::IoError(e), + ArtcraftError::StorytellerError(e) => Self::StorytellerError(e), + _ => Self::ArtcraftError(value), + } + } +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/mod.rs index e1c54ebd9e..7e71b73802 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/mod.rs @@ -1,4 +1,5 @@ pub mod enqueue_text_to_image_command; -pub mod handle_fal; -pub mod handle_sora; +pub mod handle_image_artcraft; +pub mod handle_image_fal; +pub mod handle_image_sora; pub mod internal_image_error; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/mod.rs index a91780e378..c70c64e67d 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/mod.rs @@ -1,2 +1,3 @@ pub mod image; -pub mod object; \ No newline at end of file +pub mod object; +pub mod video; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/object/enqueue_image_to_3d_object_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/object/enqueue_image_to_3d_object_command.rs index eab5e57614..01a35f0f57 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/object/enqueue_image_to_3d_object_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/object/enqueue_image_to_3d_object_command.rs @@ -1,5 +1,4 @@ -use crate::core::commands::enqueue::image::handle_fal::handle_fal; -use crate::core::commands::enqueue::image::handle_sora::handle_sora; +use crate::core::commands::enqueue::object::handle_object_artcraft::handle_object_artcraft; use crate::core::commands::enqueue::object::handle_object_fal::handle_object_fal; use crate::core::commands::enqueue::object::internal_object_error::InternalObjectError; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; @@ -25,7 +24,7 @@ use base64::prelude::BASE64_STANDARD; use base64::{DecodeError, Engine}; use errors::{AnyhowError, AnyhowResult}; use fal_client::error::fal_error_plus::FalErrorPlus; -use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; use filesys::file_read_bytes::file_read_bytes; use image::codecs::png::{CompressionType, FilterType, PngEncoder}; @@ -57,7 +56,6 @@ use storyteller_client::utils::api_host::ApiHost; use tauri::{AppHandle, Emitter, Manager, State}; use tempfile::NamedTempFile; use tokens::tokens::media_files::MediaFileToken; -use crate::core::commands::enqueue::object::handle_object_artcraft::handle_object_artcraft; #[derive(Deserialize)] pub struct EnqueueImageTo3dObjectRequest { diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/object/handle_object_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/object/handle_object_artcraft.rs index 0563d9c9be..af8ac4510d 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/object/handle_object_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/object/handle_object_artcraft.rs @@ -17,7 +17,7 @@ use anyhow::anyhow; use artcraft_api_defs::generate::object::generate_hunyuan_2_image_to_3d::GenerateHunyuan2ImageTo3dRequest; use fal_client::creds::fal_api_key::FalApiKey; use fal_client::requests::queue::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; -use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; use idempotency::uuid::generate_random_uuid; use log::{error, info, warn}; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/object/handle_object_fal.rs b/crates/desktop/artcraft/src/core/commands/enqueue/object/handle_object_fal.rs index eb26daa750..e99a3d2eda 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/object/handle_object_fal.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/object/handle_object_fal.rs @@ -16,7 +16,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use anyhow::anyhow; use fal_client::creds::fal_api_key::FalApiKey; use fal_client::requests::queue::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; -use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; use log::{error, info, warn}; use tauri::AppHandle; @@ -34,7 +34,7 @@ pub async fn handle_object_fal( None => { error!("No FAL API key is set. Can't perform action."); let event = - GenerationEnqueueFailureEvent::no_fal_api_key(GenerationAction::GenerateImage); + GenerationEnqueueFailureEvent::no_fal_api_key(GenerationAction::ImageTo3d); if let Err(err) = event.send(&app) { error!("Failed to emit event: {:?}", err); // Fail open. diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/object/internal_object_error.rs b/crates/desktop/artcraft/src/core/commands/enqueue/object/internal_object_error.rs index a85886af73..7d410d94ba 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/object/internal_object_error.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/object/internal_object_error.rs @@ -1,9 +1,7 @@ use crate::core::artcraft_error::ArtcraftError; -use anyhow::anyhow; use base64::DecodeError; use errors::AnyhowError; use fal_client::error::fal_error_plus::FalErrorPlus; -use storyteller_client::error::api_error::ApiError; use storyteller_client::error::storyteller_error::StorytellerError; #[derive(Debug)] @@ -15,9 +13,9 @@ pub enum InternalObjectError { FalError(FalErrorPlus), AnyhowError(AnyhowError), StorytellerError(StorytellerError), - StorytellerApiError(ApiError), DecodeError(DecodeError), IoError(std::io::Error), + ArtcraftError(ArtcraftError), } impl From for InternalObjectError { @@ -32,9 +30,9 @@ impl From for InternalObjectError { } } -impl From for InternalObjectError { - fn from(value: ApiError) -> Self { - Self::StorytellerApiError(value) +impl From for InternalObjectError { + fn from(value: StorytellerError) -> Self { + Self::StorytellerError(value) } } @@ -57,9 +55,7 @@ impl From for InternalObjectError { ArtcraftError::DecodeError(e) => Self::DecodeError(e), ArtcraftError::IoError(e) => Self::IoError(e), ArtcraftError::StorytellerError(e) => Self::StorytellerError(e), - _ => { - Self::AnyhowError(anyhow!("Other error: {:?}", value)) - } + _ => Self::ArtcraftError(value), } } } diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/video/enqueue_image_to_video_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/video/enqueue_image_to_video_command.rs new file mode 100644 index 0000000000..09e6fc2a77 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/video/enqueue_image_to_video_command.rs @@ -0,0 +1,179 @@ +use crate::core::commands::enqueue::object::handle_object_artcraft::handle_object_artcraft; +use crate::core::commands::enqueue::object::handle_object_fal::handle_object_fal; +use crate::core::commands::enqueue::video::handle_video_artcraft::handle_video_artcraft; +use crate::core::commands::enqueue::video::handle_video_fal::handle_video_fal; +use crate::core::commands::enqueue::video::internal_video_error::InternalVideoError; +use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; +use crate::core::commands::response::shorthand::Response; +use crate::core::commands::response::success_response_wrapper::SerializeMarker; +use crate::core::events::sendable_event_trait::SendableEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk; +use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use base64::prelude::BASE64_STANDARD; +use base64::{DecodeError, Engine}; +use errors::{AnyhowError, AnyhowResult}; +use fal_client::error::fal_error_plus::FalErrorPlus; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; +use filesys::file_read_bytes::file_read_bytes; +use image::codecs::png::{CompressionType, FilterType, PngEncoder}; +use image::{DynamicImage, EncodableLayout, ImageReader}; +use log::{debug, error, info, warn}; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use openai_sora_client::credentials::SoraCredentials; +use openai_sora_client::creds::credential_migration::CredentialMigrationRef; +use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest}; +use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; +use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes; +use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file; +use openai_sora_client::sora_error::SoraError; +use reqwest::Url; +use serde_derive::{Deserialize, Serialize}; +use std::fs::read_to_string; +use std::io::{Cursor, Read, Write}; +use std::ops::Add; +use std::path::PathBuf; +use std::time::Duration; +use storyteller_client::error::api_error::ApiError; +use storyteller_client::error::api_error::ApiError::InternalServerError; +use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse}; +use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; +use storyteller_client::utils::api_host::ApiHost; +use tauri::{AppHandle, Emitter, Manager, State}; +use tempfile::NamedTempFile; +use tokens::tokens::media_files::MediaFileToken; + +#[derive(Deserialize)] +pub struct EnqueueImageToVideoRequest { + /// Image media file; the image to remove the background from. + /// TODO: In the future we may support base64 images, URLs, or file paths here. + pub image_media_token: Option, + + /// The model to use. + pub model: Option, +} + +#[derive(Deserialize, Debug, Copy, Clone)] +#[serde(rename_all = "snake_case")] +pub enum EnqueueImageToVideoModel { + #[serde(rename = "kling1_6")] + Kling16, +} + +#[derive(Serialize)] +pub struct EnqueueImageToVideoSuccessResponse { +} + +impl SerializeMarker for EnqueueImageToVideoSuccessResponse {} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum EnqueueImageToVideoErrorType { + /// Caller didn't specify a model + ModelNotSpecified, + /// Generic server error + ServerError, + /// No Fal API key available + NeedsFalApiKey, + /// Fal had an API error + FalError, + /// Needs to be logged into Artcraft + NeedsStorytellerCredentials, +} + +#[tauri::command] +pub async fn enqueue_image_to_video_command( + app: AppHandle, + request: EnqueueImageToVideoRequest, + app_data_root: State<'_, AppDataRoot>, + fal_creds_manager: State<'_, FalCredentialManager>, + fal_task_queue: State<'_, FalTaskQueue>, + storyteller_creds_manager: State<'_, StorytellerCredentialManager>, + sora_task_queue: State<'_, SoraTaskQueue>, +) -> Response { + + info!("enqueue_image_to_video_command called"); + + let result = handle_request( + &app, + request, + &app_data_root, + &fal_creds_manager, + &storyteller_creds_manager, + &fal_task_queue, + ).await; + + match result { + Err(err) => { + error!("error: {:?}", err); + + let mut status = CommandErrorStatus::ServerError; + let mut error_type = EnqueueImageToVideoErrorType::ServerError; + let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem."; + + match err { + InternalVideoError::NoModelSpecified => { + status = CommandErrorStatus::BadRequest; + error_type = EnqueueImageToVideoErrorType::ModelNotSpecified; + error_message = "No model specified for image generation"; + } + InternalVideoError::NeedsFalApiKey => { + status = CommandErrorStatus::Unauthorized; + error_type = EnqueueImageToVideoErrorType::NeedsFalApiKey; + error_message = "You need to set a FAL api key"; + }, + InternalVideoError::NeedsStorytellerCredentials => { + status = CommandErrorStatus::Unauthorized; + error_type = EnqueueImageToVideoErrorType::NeedsStorytellerCredentials; + error_message = "You need to be logged into Artcraft."; + } + _ => {}, // Fall-through + } + + Err(CommandErrorResponseWrapper { + status, + error_message: Some(error_message.to_string()), + error_type: Some(error_type), + error_details: None, + }) + } + Ok(()) => { + Ok(EnqueueImageToVideoSuccessResponse {}.into()) + } + } +} + + +pub async fn handle_request( + app: &AppHandle, + request: EnqueueImageToVideoRequest, + app_data_root: &AppDataRoot, + fal_creds_manager: &FalCredentialManager, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_task_queue: &FalTaskQueue, +) -> Result<(), InternalVideoError> { + + if fal_creds_manager.has_apparent_api_token()? { + handle_video_fal(&app, app_data_root, request, fal_creds_manager, fal_task_queue).await?; + } else { + handle_video_artcraft(request, &app, app_data_root, storyteller_creds_manager).await?; + } + + Ok(()) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_artcraft.rs new file mode 100644 index 0000000000..c3a3ec4de9 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_artcraft.rs @@ -0,0 +1,138 @@ +use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageModel, EnqueueTextToImageRequest}; +use crate::core::commands::enqueue::object::enqueue_image_to_3d_object_command::{EnqueueImageTo3dObjectModel, EnqueueImageTo3dObjectRequest}; +use crate::core::commands::enqueue::video::enqueue_image_to_video_command::{EnqueueImageToVideoModel, EnqueueImageToVideoRequest}; +use crate::core::commands::enqueue::video::internal_video_error::InternalVideoError; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use artcraft_api_defs::generate::object::generate_hunyuan_2_image_to_3d::GenerateHunyuan2ImageTo3dRequest; +use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{GenerateKling16ProAspectRatio, GenerateKling16ProImageToVideoRequest}; +use fal_client::creds::fal_api_key::FalApiKey; +use fal_client::requests::queue::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; +use fal_client::requests::queue::video_gen::enqueue_kling_16_pro_image_to_video::Kling16ProAspectRatio; +use idempotency::uuid::generate_random_uuid; +use log::{error, info, warn}; +use storyteller_client::generate::object::generate_hunyuan2_image_to_3d::generate_hunyuan2_image_to_3d; +use storyteller_client::generate::video::generate_kling_16_pro_image_to_video::generate_kling_16_pro_image_to_video; +use storyteller_client::utils::api_host::ApiHost; +use tauri::{AppHandle, State}; + +pub async fn handle_video_artcraft( + request: EnqueueImageToVideoRequest, + app: &AppHandle, + app_data_root: &AppDataRoot, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result<(), InternalVideoError> { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + error!("No Artcraft credentials are set. Can't perform action."); + let event = + GenerationEnqueueFailureEvent::no_artcraft_credentials(GenerationAction::GenerateVideo); + + if let Err(err) = event.send(&app) { + error!("Failed to emit event: {:?}", err); // Fail open. + } + + return Err(InternalVideoError::NeedsStorytellerCredentials); + }, + }; + + //let api_key = fal_creds_manager.get_key_required() + // .map_err(|err| { + // error!("EnqueueTextToImage FAL api key required: {:?}", err); + // InternalVideoError::NeedsFalApiKey + // })?; + + + info!("Calling Artcraft image to video..."); + + + let uuid_idempotency_token = generate_random_uuid(); + + let mut selected_model = None; + + let job_token = match request.model { + None => { + return Err(InternalVideoError::NoModelSpecified); + } + Some(EnqueueImageToVideoModel::Kling16) => { + info!("enqueue Kling 1.6"); + selected_model = Some(GenerationModel::Kling1_6); + let request = GenerateKling16ProImageToVideoRequest { + uuid_idempotency_token, + media_file_token: request.image_media_token, + aspect_ratio: Some(GenerateKling16ProAspectRatio::WideSixteenNine), + prompt: None, + }; + let result = generate_kling_16_pro_image_to_video( + &ApiHost::Storyteller, + Some(&creds), + request, + ).await; + match result { + Ok(enqueued) => { + info!("Successfully enqueued Artcraft Kling 1.6 video generation"); + enqueued.inference_job_token + } + Err(err) => { + error!("Failed to use Artcraft Kling 1.6 video generation: {:?}", err); + return Err(InternalVideoError::StorytellerError(err)); + } + } + } + }; + + info!("Successfully enqueued video generation"); + + let event = GenerationEnqueueSuccessEvent { + action: GenerationAction::GenerateVideo, + service: GenerationServiceProvider::Artcraft, + model: selected_model, + }; + + if let Err(err) = event.send(app) { + error!("Failed to emit event: {:?}", err); // Fail open. + } + + //if let Err(err) = fal_task_queue.insert(&enqueued) { + // error!("Failed to enqueue task: {:?}", err); + // return Err(InternalVideoError::AnyhowError(anyhow!("Failed to enqueue task: {:?}", err))); + //} + +// match result { +// Ok(enqueued) => { +// } +// Err(err) => { +// error!("Failed to enqueue image to 3d: {:?}", err); +// +// let event = GenerationEnqueueFailureEvent { +// action: GenerationAction::ImageTo3d, +// service: GenerationServiceProvider::Fal, +// model: None, +// reason: None, +// }; +// +// if let Err(err) = event.send(app) { +// error!("Failed to emit event: {:?}", err); // Fail open. +// } +// +// return Err(InternalVideoError::FalError(err)); +// } +// } + + Ok(()) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_fal.rs b/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_fal.rs new file mode 100644 index 0000000000..3204a7b154 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_fal.rs @@ -0,0 +1,124 @@ +use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageModel, EnqueueTextToImageRequest}; +use crate::core::commands::enqueue::video::enqueue_image_to_video_command::{EnqueueImageToVideoModel, EnqueueImageToVideoRequest}; +use crate::core::commands::enqueue::video::internal_video_error::InternalVideoError; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use fal_client::creds::fal_api_key::FalApiKey; +use fal_client::requests::queue::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; +use fal_client::requests::queue::video_gen::enqueue_kling_16_pro_image_to_video::{enqueue_kling_16_pro_image_to_video, Kling16ProArgs, Kling16ProAspectRatio, Kling16ProDuration}; +use log::{error, info, warn}; +use tauri::AppHandle; + +pub async fn handle_video_fal( + app: &AppHandle, + app_data_root: &AppDataRoot, + request: EnqueueImageToVideoRequest, + fal_creds_manager: &FalCredentialManager, + fal_task_queue: &FalTaskQueue, +) -> Result<(), InternalVideoError> { + + let api_key = match fal_creds_manager.get_key()? { + Some(key) => key, + None => { + error!("No FAL API key is set. Can't perform action."); + let event = + GenerationEnqueueFailureEvent::no_fal_api_key(GenerationAction::GenerateVideo); + + if let Err(err) = event.send(&app) { + error!("Failed to emit event: {:?}", err); // Fail open. + } + + return Err(InternalVideoError::NeedsFalApiKey); + }, + }; + + //let api_key = fal_creds_manager.get_key_required() + // .map_err(|err| { + // error!("EnqueueTextToImage FAL api key required: {:?}", err); + // InternalVideoError::NeedsFalApiKey + // })?; + + let mut temp_download; + + if let Some(media_token) = request.image_media_token { + temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?; + + } else { + return Err(InternalVideoError::AnyhowError(anyhow!("No image media token provided"))); + } + + info!("Calling FAL image to 3d ..."); + + let filename = temp_download.path().to_path_buf(); + + let mut selected_model = None; + + let result = match request.model { + None => { + return Err(InternalVideoError::NoModelSpecified); + } + Some(EnqueueImageToVideoModel::Kling16) => { + info!("enqueue Kling 1.6 image to video"); + selected_model = Some(GenerationModel::Kling1_6); + enqueue_kling_16_pro_image_to_video(Kling16ProArgs { + image_path: filename, + api_key: &api_key, + duration: Kling16ProDuration::Default, + aspect_ratio: Kling16ProAspectRatio::WideSixteenNine, + prompt: "", + }).await + } + }; + + match result { + Ok(enqueued) => { + info!("Successfully enqueued Kling 1.6 image to video"); + + let event = GenerationEnqueueSuccessEvent { + action: GenerationAction::GenerateVideo, + service: GenerationServiceProvider::Fal, + model: selected_model, + }; + + if let Err(err) = event.send(app) { + error!("Failed to emit event: {:?}", err); // Fail open. + } + + if let Err(err) = fal_task_queue.insert(&enqueued) { + error!("Failed to enqueue task: {:?}", err); + return Err(InternalVideoError::AnyhowError(anyhow!("Failed to enqueue task: {:?}", err))); + } + } + Err(err) => { + error!("Failed to enqueue image to video: {:?}", err); + + let event = GenerationEnqueueFailureEvent { + action: GenerationAction::GenerateVideo, + service: GenerationServiceProvider::Fal, + model: selected_model, + reason: None, + }; + + if let Err(err) = event.send(app) { + error!("Failed to emit event: {:?}", err); // Fail open. + } + + return Err(InternalVideoError::FalError(err)); + } + } + + Ok(()) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/video/internal_video_error.rs b/crates/desktop/artcraft/src/core/commands/enqueue/video/internal_video_error.rs new file mode 100644 index 0000000000..0973c58231 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/video/internal_video_error.rs @@ -0,0 +1,61 @@ +use crate::core::artcraft_error::ArtcraftError; +use base64::DecodeError; +use errors::AnyhowError; +use fal_client::error::fal_error_plus::FalErrorPlus; +use storyteller_client::error::storyteller_error::StorytellerError; + +#[derive(Debug)] +pub enum InternalVideoError { + NoModelSpecified, + NeedsFalApiKey, + NeedsSoraCredentials, + NeedsStorytellerCredentials, + FalError(FalErrorPlus), + AnyhowError(AnyhowError), + StorytellerError(StorytellerError), + DecodeError(DecodeError), + IoError(std::io::Error), + ArtcraftError(ArtcraftError), +} + +impl From for InternalVideoError { + fn from(value: AnyhowError) -> Self { + Self::AnyhowError(value) + } +} + +impl From for InternalVideoError { + fn from(value: FalErrorPlus) -> Self { + Self::FalError(value) + } +} + +impl From for InternalVideoError { + fn from(value: StorytellerError) -> Self { + Self::StorytellerError(value) + } +} + +impl From for InternalVideoError { + fn from(value: DecodeError) -> Self { + Self::DecodeError(value) + } +} + +impl From for InternalVideoError { + fn from(value: std::io::Error) -> Self { + Self::IoError(value) + } +} + +impl From for InternalVideoError { + fn from(value: ArtcraftError) -> Self { + match value { + ArtcraftError::AnyhowError(e) => Self::AnyhowError(e), + ArtcraftError::DecodeError(e) => Self::DecodeError(e), + ArtcraftError::IoError(e) => Self::IoError(e), + ArtcraftError::StorytellerError(e) => Self::StorytellerError(e), + _ => Self::ArtcraftError(value), + } + } +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/video/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/video/mod.rs new file mode 100644 index 0000000000..c61b63d3ca --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/video/mod.rs @@ -0,0 +1,4 @@ +pub mod enqueue_image_to_video_command; +pub mod handle_video_artcraft; +pub mod handle_video_fal; +pub mod internal_video_error; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/events/generation_events/common.rs b/crates/desktop/artcraft/src/core/events/generation_events/common.rs index 19649aebb8..3ec6d7f697 100644 --- a/crates/desktop/artcraft/src/core/events/generation_events/common.rs +++ b/crates/desktop/artcraft/src/core/events/generation_events/common.rs @@ -11,6 +11,8 @@ pub enum GenerationServiceProvider { #[derive(Debug, Copy, Clone, Serialize)] #[serde(rename_all = "snake_case")] pub enum GenerationModel { + #[serde(rename = "flux_pro_1.1_ultra")] + FluxPro11Ultra, #[serde(rename = "kling_1.6")] Kling1_6, #[serde(rename = "kling_2.0")] diff --git a/crates/desktop/artcraft/src/lib.rs b/crates/desktop/artcraft/src/lib.rs index 2eb80820a9..e7791f5ccf 100644 --- a/crates/desktop/artcraft/src/lib.rs +++ b/crates/desktop/artcraft/src/lib.rs @@ -7,6 +7,7 @@ use crate::core::commands::app_preferences::get_app_preferences_command::get_app use crate::core::commands::app_preferences::update_app_preference_command::update_app_preferences_command; use crate::core::commands::enqueue::image::enqueue_text_to_image_command::enqueue_text_to_image_command; use crate::core::commands::enqueue::object::enqueue_image_to_3d_object_command::enqueue_image_to_3d_object_command; +use crate::core::commands::enqueue::video::enqueue_image_to_video_command::enqueue_image_to_video_command; use crate::core::commands::flip_image::flip_image; use crate::core::commands::get_build_info_command::get_build_info_command; use crate::core::commands::platform_info_command::platform_info_command; @@ -163,6 +164,7 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ check_sora_session_command, enqueue_image_to_3d_object_command, + enqueue_image_to_video_command, enqueue_text_to_image_command, fal_background_removal_command, fal_hunyuan_image_to_3d_command, diff --git a/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs b/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs index 828dd0fc24..3dbd65c081 100644 --- a/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs +++ b/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs @@ -1,3 +1,4 @@ +use crate::core::artcraft_error::ArtcraftError; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::commands::response::shorthand::Response; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; @@ -24,7 +25,7 @@ use base64::prelude::BASE64_STANDARD; use base64::{DecodeError, Engine}; use errors::{AnyhowError, AnyhowResult}; use fal_client::error::fal_error_plus::FalErrorPlus; -use fal_client::requests::queue::video_gen::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration}; +use fal_client::requests::queue::video_gen::enqueue_kling_16_pro_image_to_video::{enqueue_kling_16_pro_image_to_video, Kling16ProArgs, Kling16ProAspectRatio, Kling16ProDuration}; use filesys::file_read_bytes::file_read_bytes; use image::codecs::png::{CompressionType, FilterType, PngEncoder}; use image::{DynamicImage, EncodableLayout, ImageReader}; @@ -49,14 +50,13 @@ use std::path::PathBuf; use std::time::Duration; use storyteller_client::error::api_error::ApiError; use storyteller_client::error::api_error::ApiError::InternalServerError; +use storyteller_client::error::storyteller_error::StorytellerError; use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse}; use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; use storyteller_client::utils::api_host::ApiHost; use tauri::{AppHandle, Emitter, Manager, State}; use tempfile::NamedTempFile; -use storyteller_client::error::storyteller_error::StorytellerError; use tokens::tokens::media_files::MediaFileToken; -use crate::core::artcraft_error::ArtcraftError; #[derive(Deserialize)] pub struct FalKlingImageToVideoRequest { @@ -233,10 +233,11 @@ pub async fn image_to_video( let filename = temp_download.path().to_path_buf(); - let enqueued = enqueue_kling_16_image_to_video(Kling16Args { + let enqueued = enqueue_kling_16_pro_image_to_video(Kling16ProArgs { image_path: filename, api_key: &api_key, - duration: Kling16Duration::Default, + duration: Kling16ProDuration::Default, + aspect_ratio: Kling16ProAspectRatio::WideSixteenNine, prompt: request.prompt.as_deref().unwrap_or(""), }).await?; diff --git a/crates/lib/api_clients/fal_client/src/requests/queue/image_gen/enqueue_flux_pro_ultra_text_to_image.rs b/crates/lib/api_clients/fal_client/src/requests/queue/image_gen/enqueue_flux_pro_11_ultra_text_to_image.rs similarity index 77% rename from crates/lib/api_clients/fal_client/src/requests/queue/image_gen/enqueue_flux_pro_ultra_text_to_image.rs rename to crates/lib/api_clients/fal_client/src/requests/queue/image_gen/enqueue_flux_pro_11_ultra_text_to_image.rs index e677082a51..819eb62d06 100644 --- a/crates/lib/api_clients/fal_client/src/requests/queue/image_gen/enqueue_flux_pro_ultra_text_to_image.rs +++ b/crates/lib/api_clients/fal_client/src/requests/queue/image_gen/enqueue_flux_pro_11_ultra_text_to_image.rs @@ -7,22 +7,25 @@ use fal::endpoints::fal_ai::flux_pro::v1_1_ultra::FluxProUltraTextToImageInput; use futures::StreamExt; use std::io::Write; -pub struct FluxProUltraTextToImageArgs<'a> { +pub struct FluxPro11UltraTextToImageArgs<'a> { pub prompt: &'a str, pub api_key: &'a FalApiKey, // TODO: Size parameter } -pub async fn enqueue_flux_pro_ultra_text_to_image(args: FluxProUltraTextToImageArgs<'_>) -> Result { +pub async fn enqueue_flux_pro_11_ultra_text_to_image(args: FluxPro11UltraTextToImageArgs<'_>) -> Result { let request = FluxProUltraTextToImageInput { prompt: args.prompt.to_string(), - safety_tolerance: Some("5".to_string()), // 1 is most strict, 5 is most permissive - enable_safety_checker: Some(false), - num_images: Some(1), // Default is 1 - output_format: Some("png".to_string()), - raw: Some(true), // Generate less processed, more natural-looking images. Default is false. + // Maybe expose aspect_ratio: None, // Default is "16:9" + num_images: Some(1), // Default is 1 seed: None, + raw: Some(true), // Generate less processed, more natural-looking images. Default is false. + // Maybe abstract + enable_safety_checker: Some(false), + safety_tolerance: Some("5".to_string()), // 1 is most strict, 5 is most permissive + // Constants + output_format: Some("png".to_string()), sync_mode: None, // Synchronous / slow }; @@ -42,7 +45,7 @@ pub async fn enqueue_flux_pro_ultra_text_to_image(args: FluxProUltraTextToImageA #[cfg(test)] mod tests { use crate::creds::fal_api_key::FalApiKey; - use crate::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs}; + use crate::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use errors::AnyhowResult; use std::fs::read_to_string; @@ -54,12 +57,12 @@ mod tests { let api_key = FalApiKey::from_str(&secret); - let args = FluxProUltraTextToImageArgs { + let args = FluxPro11UltraTextToImageArgs { prompt: "a corgi smiles at the camera, inside retro video game store, lots of sci-fi props from Cowboy Bebop, photorealistic", api_key: &api_key, }; - let result = enqueue_flux_pro_ultra_text_to_image(args).await?; + let result = enqueue_flux_pro_11_ultra_text_to_image(args).await?; Ok(()) } diff --git a/crates/lib/api_clients/fal_client/src/requests/queue/image_gen/mod.rs b/crates/lib/api_clients/fal_client/src/requests/queue/image_gen/mod.rs index 6716f1cdf0..440fb491c0 100644 --- a/crates/lib/api_clients/fal_client/src/requests/queue/image_gen/mod.rs +++ b/crates/lib/api_clients/fal_client/src/requests/queue/image_gen/mod.rs @@ -1,2 +1,2 @@ -pub mod enqueue_flux_pro_ultra_text_to_image; +pub mod enqueue_flux_pro_11_ultra_text_to_image; pub mod enqueue_recraft3_text_to_image; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/requests/queue/video_gen/enqueue_kling_16_image_to_video.rs b/crates/lib/api_clients/fal_client/src/requests/queue/video_gen/enqueue_kling_16_pro_image_to_video.rs similarity index 57% rename from crates/lib/api_clients/fal_client/src/requests/queue/video_gen/enqueue_kling_16_image_to_video.rs rename to crates/lib/api_clients/fal_client/src/requests/queue/video_gen/enqueue_kling_16_pro_image_to_video.rs index a664299a06..c07455c22d 100644 --- a/crates/lib/api_clients/fal_client/src/requests/queue/video_gen/enqueue_kling_16_image_to_video.rs +++ b/crates/lib/api_clients/fal_client/src/requests/queue/video_gen/enqueue_kling_16_pro_image_to_video.rs @@ -8,35 +8,50 @@ use futures::StreamExt; use std::io::Write; use std::path::Path; -pub struct Kling16Args<'a, P: AsRef> { +pub struct Kling16ProArgs<'a, P: AsRef> { pub image_path: P, pub prompt: &'a str, pub api_key: &'a FalApiKey, - pub duration: Kling16Duration, + pub duration: Kling16ProDuration, + pub aspect_ratio: Kling16ProAspectRatio, } #[derive(Copy, Clone, Debug)] -pub enum Kling16Duration { +pub enum Kling16ProDuration { Default, FiveSeconds, TenSeconds, } -pub async fn enqueue_kling_16_image_to_video>(args: Kling16Args<'_, P>) -> Result { +#[derive(Copy, Clone, Debug)] +pub enum Kling16ProAspectRatio { + Square, // 1:1 + WideSixteenNine, // 16:9 + TallNineSixteen, // 9:16 +} + +pub async fn enqueue_kling_16_pro_image_to_video>(args: Kling16ProArgs<'_, P>) -> Result { let image_url = file_to_base64_url(args.image_path)?; let duration = match args.duration{ - Kling16Duration::Default => None, - Kling16Duration::FiveSeconds => Some("5".to_string()), // Gross... - Kling16Duration::TenSeconds => Some("10".to_string()), + Kling16ProDuration::Default => None, + Kling16ProDuration::FiveSeconds => Some("5".to_string()), // Gross... + Kling16ProDuration::TenSeconds => Some("10".to_string()), + }; + + let aspect_ratio = match args.aspect_ratio { + Kling16ProAspectRatio::Square => Some("1:1".to_string()), + Kling16ProAspectRatio::WideSixteenNine => Some("16:9".to_string()), + Kling16ProAspectRatio::TallNineSixteen => Some("9:16".to_string()), }; let request = ProImageToVideoRequest { image_url, prompt: args.prompt.to_string(), - aspect_ratio: None, - cfg_scale: None, + aspect_ratio, duration, + // Maybe expose these later + cfg_scale: None, negative_prompt: None, tail_image_url: None, }; @@ -57,7 +72,7 @@ pub async fn enqueue_kling_16_image_to_video>(args: Kling16Args<' #[cfg(test)] mod tests { use crate::creds::fal_api_key::FalApiKey; - use crate::requests::queue::video_gen::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration}; + use crate::requests::queue::video_gen::enqueue_kling_16_pro_image_to_video::{enqueue_kling_16_pro_image_to_video, Kling16ProArgs, Kling16ProAspectRatio, Kling16ProDuration}; use errors::AnyhowResult; use std::fs::read_to_string; use testing::test_file_path::test_file_path; @@ -72,14 +87,15 @@ mod tests { let api_key = FalApiKey::from_str(&secret); - let args = Kling16Args { + let args = Kling16ProArgs { image_path: image, prompt: "a corgi looks out over the water", api_key: &api_key, - duration: Kling16Duration::Default, + duration: Kling16ProDuration::Default, + aspect_ratio: Kling16ProAspectRatio::Square, }; - let result = enqueue_kling_16_image_to_video(args).await?; + let result = enqueue_kling_16_pro_image_to_video(args).await?; Ok(()) } diff --git a/crates/lib/api_clients/fal_client/src/requests/queue/video_gen/mod.rs b/crates/lib/api_clients/fal_client/src/requests/queue/video_gen/mod.rs index 7776327a79..228901495d 100644 --- a/crates/lib/api_clients/fal_client/src/requests/queue/video_gen/mod.rs +++ b/crates/lib/api_clients/fal_client/src/requests/queue/video_gen/mod.rs @@ -1,2 +1,2 @@ -pub mod enqueue_kling_16_image_to_video; +pub mod enqueue_kling_16_pro_image_to_video; pub mod enqueue_minimax_01_image_to_video; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_1_dev_text_to_image_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_1_dev_text_to_image_webhook.rs new file mode 100644 index 0000000000..d7dbc4c0ca --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_1_dev_text_to_image_webhook.rs @@ -0,0 +1,106 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::flux::dev::dev; +use fal::endpoints::fal_ai::flux::dev::DevTextToImageInput; +use fal::prelude::fal_ai::flux::dev::ImageSizeProperty; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Flux1DevArgs<'a, U: IntoUrl> { + pub prompt: &'a str, + pub webhook_url: U, + pub api_key: &'a FalApiKey, + pub aspect_ratio: Flux1DevAspectRatio, + pub num_images: Flux1DevNumImages, +} + +#[derive(Copy, Clone, Debug)] +pub enum Flux1DevAspectRatio { + Square, // 1:1 + SquareHd, // 1:1 + LandscapeFourByThree, // 4:3 + LandscapeSixteenByNine, // 16:9 + PortraitThreeByFour, // 3:4 + PortraitNineBySixteen, // 9:16 + //Custom { width: u32, height: u32 }, // TODO +} + +#[derive(Copy, Clone, Debug)] +pub enum Flux1DevNumImages { + One, // Default + Two, + Three, + Four, +} + +pub async fn enqueue_flux_1_dev_text_to_image_webhook( + args: Flux1DevArgs<'_, U> +) -> Result { + + let num_images = match args.num_images { + Flux1DevNumImages::One => 1, + Flux1DevNumImages::Two => 2, + Flux1DevNumImages::Three => 3, + Flux1DevNumImages::Four => 4, + }; + + let image_size = match args.aspect_ratio { + Flux1DevAspectRatio::Square => ImageSizeProperty::Square, + Flux1DevAspectRatio::SquareHd => ImageSizeProperty::SquareHd, + Flux1DevAspectRatio::LandscapeFourByThree => ImageSizeProperty::Landscape43, + Flux1DevAspectRatio::LandscapeSixteenByNine => ImageSizeProperty::Landscape169, + Flux1DevAspectRatio::PortraitThreeByFour => ImageSizeProperty::Portrait43, + Flux1DevAspectRatio::PortraitNineBySixteen => ImageSizeProperty::Portrait169, + }; + + let request = DevTextToImageInput { + prompt: args.prompt.to_string(), + num_images: Some(num_images), + image_size: Some(image_size), + // Maybe abstract + enable_safety_checker: Some(false), + // Maybe expose + seed: None, + guidance_scale: None, + num_inference_steps: None, + // Constants + sync_mode: None, // Synchronous / slow + }; + + let result = dev(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::image::enqueue_flux_1_dev_text_to_image_webhook::{enqueue_flux_1_dev_text_to_image_webhook, Flux1DevArgs, Flux1DevAspectRatio, Flux1DevNumImages}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_flux_1_dev() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Flux1DevArgs { + prompt: "a giant robot fighting a dragon in a futuristic city", + api_key: &api_key, + webhook_url: "https://example.com/webhook", + num_images: Flux1DevNumImages::One, + aspect_ratio: Flux1DevAspectRatio::LandscapeFourByThree + }; + + let result = enqueue_flux_1_dev_text_to_image_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_1_schnell_text_to_image_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_1_schnell_text_to_image_webhook.rs new file mode 100644 index 0000000000..8a2560a1ad --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_1_schnell_text_to_image_webhook.rs @@ -0,0 +1,105 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::flux::schnell::schnell; +use fal::endpoints::fal_ai::flux::schnell::SchnellTextToImageInput; +use fal::prelude::fal_ai::flux::schnell::ImageSizeProperty; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Flux1SchnellArgs<'a, U: IntoUrl> { + pub prompt: &'a str, + pub webhook_url: U, + pub api_key: &'a FalApiKey, + pub aspect_ratio: Flux1SchnellAspectRatio, + pub num_images: Flux1SchnellNumImages, +} + +#[derive(Copy, Clone, Debug)] +pub enum Flux1SchnellAspectRatio { + Square, // 1:1 + SquareHd, // 1:1 + LandscapeFourByThree, // 4:3 + LandscapeSixteenByNine, // 16:9 + PortraitThreeByFour, // 3:4 + PortraitNineBySixteen, // 9:16 + //Custom { width: u32, height: u32 }, // TODO +} + +#[derive(Copy, Clone, Debug)] +pub enum Flux1SchnellNumImages { + One, // Default + Two, + Three, + Four, +} + +pub async fn enqueue_flux_1_schnell_text_to_image_webhook( + args: Flux1SchnellArgs<'_, U> +) -> Result { + + let num_images = match args.num_images { + Flux1SchnellNumImages::One => 1, + Flux1SchnellNumImages::Two => 2, + Flux1SchnellNumImages::Three => 3, + Flux1SchnellNumImages::Four => 4, + }; + + let image_size = match args.aspect_ratio { + Flux1SchnellAspectRatio::Square => ImageSizeProperty::Square, + Flux1SchnellAspectRatio::SquareHd => ImageSizeProperty::SquareHd, + Flux1SchnellAspectRatio::LandscapeFourByThree => ImageSizeProperty::Landscape43, + Flux1SchnellAspectRatio::LandscapeSixteenByNine => ImageSizeProperty::Landscape169, + Flux1SchnellAspectRatio::PortraitThreeByFour => ImageSizeProperty::Portrait43, + Flux1SchnellAspectRatio::PortraitNineBySixteen => ImageSizeProperty::Portrait169, + }; + + let request = SchnellTextToImageInput { + prompt: args.prompt.to_string(), + num_images: Some(num_images), + image_size: Some(image_size), + // Maybe abstract + enable_safety_checker: Some(false), + // Maybe expose + seed: None, + num_inference_steps: None, + // Constants + sync_mode: None, // Synchronous / slow + }; + + let result = schnell(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::image::enqueue_flux_1_schnell_text_to_image_webhook::{enqueue_flux_1_schnell_text_to_image_webhook, Flux1SchnellArgs, Flux1SchnellAspectRatio, Flux1SchnellNumImages}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_flux_1_schnell() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Flux1SchnellArgs { + prompt: "a giant robot fighting a dragon in a futuristic city", + api_key: &api_key, + webhook_url: "https://example.com/webhook", + num_images: Flux1SchnellNumImages::One, + aspect_ratio: Flux1SchnellAspectRatio::LandscapeSixteenByNine, + }; + + let result = enqueue_flux_1_schnell_text_to_image_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_pro_11_text_to_image_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_pro_11_text_to_image_webhook.rs new file mode 100644 index 0000000000..85b182a7df --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_pro_11_text_to_image_webhook.rs @@ -0,0 +1,104 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::flux_pro::v1_1::{v1_1, FluxProPlusTextToImageInput, ImageSizeProperty}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct FluxPro11Args<'a, U: IntoUrl> { + pub prompt: &'a str, + pub webhook_url: U, + pub api_key: &'a FalApiKey, + pub aspect_ratio: FluxPro11AspectRatio, + pub num_images: FluxPro11NumImages, +} + +#[derive(Copy, Clone, Debug)] +pub enum FluxPro11AspectRatio { + Square, // 1:1 + SquareHd, // 1:1 + LandscapeFourByThree, // 4:3 + LandscapeSixteenByNine, // 16:9 + PortraitThreeByFour, // 3:4 + PortraitNineBySixteen, // 9:16 + //Custom { width: u32, height: u32 }, // TODO +} + +#[derive(Copy, Clone, Debug)] +pub enum FluxPro11NumImages { + One, // Default + Two, + Three, + Four, +} + +pub async fn enqueue_flux_pro_11_text_to_image_webhook( + args: FluxPro11Args<'_, U> +) -> Result { + + let num_images = match args.num_images { + FluxPro11NumImages::One => 1, + FluxPro11NumImages::Two => 2, + FluxPro11NumImages::Three => 3, + FluxPro11NumImages::Four => 4, + }; + + let image_size = match args.aspect_ratio { + FluxPro11AspectRatio::Square => ImageSizeProperty::Square, + FluxPro11AspectRatio::SquareHd => ImageSizeProperty::SquareHd, + FluxPro11AspectRatio::LandscapeFourByThree => ImageSizeProperty::Landscape43, + FluxPro11AspectRatio::LandscapeSixteenByNine => ImageSizeProperty::Landscape169, + FluxPro11AspectRatio::PortraitThreeByFour => ImageSizeProperty::Portrait43, + FluxPro11AspectRatio::PortraitNineBySixteen => ImageSizeProperty::Portrait169, + }; + + let request = FluxProPlusTextToImageInput { + prompt: args.prompt.to_string(), + num_images: Some(num_images), + image_size: Some(image_size), + // Maybe expose + seed: None, + // Maybe abstract + enable_safety_checker: Some(false), + safety_tolerance: Some("5".to_string()), // 1 is most strict, 5 is most permissive + // Constants + output_format: Some("png".to_string()), + sync_mode: None, // Synchronous / slow + }; + + let result = v1_1(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::image::enqueue_flux_pro_11_text_to_image_webhook::{enqueue_flux_pro_11_text_to_image_webhook, FluxPro11Args, FluxPro11AspectRatio, FluxPro11NumImages}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_flux_11_pro() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = FluxPro11Args { + prompt: "a giant red panda fighting a dragon in a futuristic city", + api_key: &api_key, + webhook_url: "https://example.com/webhook", + num_images: FluxPro11NumImages::One, + aspect_ratio: FluxPro11AspectRatio::LandscapeSixteenByNine, + }; + + let result = enqueue_flux_pro_11_text_to_image_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_pro_11_ultra_text_to_image_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_pro_11_ultra_text_to_image_webhook.rs new file mode 100644 index 0000000000..51946abe20 --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/image/enqueue_flux_pro_11_ultra_text_to_image_webhook.rs @@ -0,0 +1,39 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::flux_pro::v1_1_ultra::{v1_1_ultra, FluxProUltraTextToImageInput}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct FluxPro11UltraArgs<'a, U: IntoUrl> { + pub prompt: &'a str, + pub webhook_url: U, + pub api_key: &'a FalApiKey, +} + +pub async fn enqueue_flux_pro_11_ultra_text_to_image_webhook( + args: FluxPro11UltraArgs<'_, U> +) -> Result { + + let request = FluxProUltraTextToImageInput { + prompt: args.prompt.to_string(), + // Maybe expose + num_images: Some(1), // Default is 1 + aspect_ratio: None, // Default is "16:9" + seed: None, + raw: Some(true), // Generate less processed, more natural-looking images. Default is false. + // Maybe abstract + enable_safety_checker: Some(false), + safety_tolerance: Some("5".to_string()), // 1 is most strict, 5 is most permissive + // Constants + output_format: Some("png".to_string()), + sync_mode: None, // Synchronous / slow + }; + + let result = v1_1_ultra(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/image/mod.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/image/mod.rs index ea3c2d1bee..15acbf30b6 100644 --- a/crates/lib/api_clients/fal_client/src/requests/webhook/image/mod.rs +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/image/mod.rs @@ -1 +1,5 @@ +pub mod enqueue_flux_1_dev_text_to_image_webhook; +pub mod enqueue_flux_1_schnell_text_to_image_webhook; +pub mod enqueue_flux_pro_11_text_to_image_webhook; +pub mod enqueue_flux_pro_11_ultra_text_to_image_webhook; pub mod remove_background_rembg_webhook; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan_3d_21_image_to_3d_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan_3d_21_image_to_3d_webhook.rs new file mode 100644 index 0000000000..b599151b90 --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan_3d_21_image_to_3d_webhook.rs @@ -0,0 +1,65 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::prelude::fal_ai::hunyuan3d::v21::{v21, Hunyuan3DInput}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Hunyuan3d21Args<'a, U: IntoUrl, V: IntoUrl> { + pub image_url: U, + pub webhook_url: V, + pub api_key: &'a FalApiKey, +} + +pub async fn enqueue_hunyuan_3d_2_1_image_to_3d_webhook( + args: Hunyuan3d21Args<'_, U, V> +) -> Result { + + let image_url = args.image_url.as_str().to_string(); + + let request = Hunyuan3DInput { + input_image_url: image_url, + textured_mesh: Some(true), + // TODO: Maybe expose these later + guidance_scale: None, + num_inference_steps: None, + octree_resolution: None, + seed: None, + }; + + let result = v21(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::object::enqueue_hunyuan_3d_21_image_to_3d_webhook::{enqueue_hunyuan_3d_2_1_image_to_3d_webhook, Hunyuan3d21Args}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_hunyuan3d_21() -> AnyhowResult<()> { + let image_url = "https://cdn-2.fakeyou.com/media/p/a/c/7/j/pac7jgp2tkehm7j7sm4sky1fpnkrnbve/image_pac7jgp2tkehm7j7sm4sky1fpnkrnbve.png"; + + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Hunyuan3d21Args { + image_url: image_url, + api_key: &api_key, + webhook_url: "https://example.com/webhook", + }; + + let result = enqueue_hunyuan_3d_2_1_image_to_3d_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan2_image_to_3d_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan_3d_2_image_to_3d_webhook.rs similarity index 75% rename from crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan2_image_to_3d_webhook.rs rename to crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan_3d_2_image_to_3d_webhook.rs index 7a7cf296e5..57715bab88 100644 --- a/crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan2_image_to_3d_webhook.rs +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/object/enqueue_hunyuan_3d_2_image_to_3d_webhook.rs @@ -5,21 +5,14 @@ use fal::endpoints::fal_ai::hunyuan3d::v2::{v2, Hunyuan3DInput}; use fal::webhook::WebhookResponse; use reqwest::IntoUrl; -pub struct Hunyuan2Args<'a, U: IntoUrl, V: IntoUrl> { +pub struct Hunyuan3d2Args<'a, U: IntoUrl, V: IntoUrl> { pub image_url: U, pub webhook_url: V, pub api_key: &'a FalApiKey, } -#[derive(Copy, Clone, Debug)] -pub enum Kling16Duration { - Default, - FiveSeconds, - TenSeconds, -} - -pub async fn enqueue_hunyuan2_image_to_3d_webhook( - args: Hunyuan2Args<'_, U, V> +pub async fn enqueue_hunyuan_3d_2_image_to_3d_webhook( + args: Hunyuan3d2Args<'_, U, V> ) -> Result { let image_url = args.image_url.as_str().to_string(); diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/object/mod.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/object/mod.rs index ca65e4d859..c818cc0721 100644 --- a/crates/lib/api_clients/fal_client/src/requests/webhook/object/mod.rs +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/object/mod.rs @@ -1 +1,2 @@ -pub mod enqueue_hunyuan2_image_to_3d_webhook; \ No newline at end of file +pub mod enqueue_hunyuan_3d_2_image_to_3d_webhook; +pub mod enqueue_hunyuan_3d_21_image_to_3d_webhook; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_image_to_video_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs similarity index 76% rename from crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_image_to_video_webhook.rs rename to crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs index 345c12c5da..9e6f9a72a3 100644 --- a/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_image_to_video_webhook.rs +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs @@ -5,12 +5,13 @@ use fal::endpoints::fal_ai::kling_video::v1_6::pro::image_to_video::{image_to_vi use fal::webhook::WebhookResponse; use reqwest::IntoUrl; -pub struct Kling16Args<'a, U: IntoUrl, V: IntoUrl> { +pub struct Kling16ProArgs<'a, U: IntoUrl, V: IntoUrl> { pub image_url: U, pub webhook_url: V, pub prompt: &'a str, pub api_key: &'a FalApiKey, pub duration: Kling16Duration, + pub aspect_ratio: Kling16ProAspectRatio, } #[derive(Copy, Clone, Debug)] @@ -20,8 +21,15 @@ pub enum Kling16Duration { TenSeconds, } -pub async fn enqueue_kling_16_image_to_video_webhook( - args: Kling16Args<'_, U, V> +#[derive(Copy, Clone, Debug)] +pub enum Kling16ProAspectRatio { + Square, // 1:1 + WideSixteenNine, // 16:9 + TallNineSixteen, // 9:16 +} + +pub async fn enqueue_kling_16_pro_image_to_video_webhook( + args: Kling16ProArgs<'_, U, V> ) -> Result { let duration = match args.duration { Kling16Duration::Default => None, @@ -29,14 +37,21 @@ pub async fn enqueue_kling_16_image_to_video_webhook( Kling16Duration::TenSeconds => Some("10".to_string()), }; + let aspect_ratio = match args.aspect_ratio { + Kling16ProAspectRatio::Square => Some("1:1".to_string()), + Kling16ProAspectRatio::WideSixteenNine => Some("16:9".to_string()), + Kling16ProAspectRatio::TallNineSixteen => Some("9:16".to_string()), + }; + let image_url = args.image_url.as_str().to_string(); let request = ProImageToVideoRequest { image_url, prompt: args.prompt.to_string(), - aspect_ratio: None, - cfg_scale: None, + aspect_ratio, duration, + // Maybe expose these later + cfg_scale: None, negative_prompt: None, tail_image_url: None, }; diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_master_image_to_video_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_master_image_to_video_webhook.rs new file mode 100644 index 0000000000..bdd88d07fe --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_master_image_to_video_webhook.rs @@ -0,0 +1,98 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::kling_video::v2_1::master::image_to_video::{image_to_video, MasterImageToVideoRequest}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Kling21MasterArgs<'a, U: IntoUrl, V: IntoUrl> { + pub image_url: U, + pub webhook_url: V, + pub prompt: &'a str, + pub api_key: &'a FalApiKey, + pub duration: Kling21MasterDuration, + pub aspect_ratio: Kling21MasterAspectRatio, +} + +#[derive(Copy, Clone, Debug)] +pub enum Kling21MasterDuration { + Default, + FiveSeconds, + TenSeconds, +} + +#[derive(Copy, Clone, Debug)] +pub enum Kling21MasterAspectRatio { + Square, // 1:1 + WideSixteenNine, // 16:9 + TallNineSixteen, // 9:16 +} + +pub async fn enqueue_kling_21_master_image_to_video_webhook( + args: Kling21MasterArgs<'_, U, V> +) -> Result { + let duration = match args.duration { + Kling21MasterDuration::Default => None, + Kling21MasterDuration::FiveSeconds => Some("5".to_string()), // Gross... + Kling21MasterDuration::TenSeconds => Some("10".to_string()), + }; + + let aspect_ratio = match args.aspect_ratio { + Kling21MasterAspectRatio::Square => Some("1:1".to_string()), + Kling21MasterAspectRatio::WideSixteenNine => Some("16:9".to_string()), + Kling21MasterAspectRatio::TallNineSixteen => Some("9:16".to_string()), + }; + + let image_url = args.image_url.as_str().to_string(); + + let request = MasterImageToVideoRequest { + image_url, + prompt: args.prompt.to_string(), + aspect_ratio, + duration, + // Maybe expose these later + cfg_scale: None, + negative_prompt: None, + tail_image_url: None, + }; + + let result = image_to_video(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::video::enqueue_kling_21_master_image_to_video_webhook::{enqueue_kling_21_master_image_to_video_webhook, Kling21MasterArgs, Kling21MasterAspectRatio, Kling21MasterDuration}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_kling21_master_video() -> AnyhowResult<()> { + let image_url = "https://cdn-2.fakeyou.com/media/3/4/h/f/s/34hfsmt8e38rvne6mwa4pwbxr6292sgy/image_34hfsmt8e38rvne6mwa4pwbxr6292sgy.png"; + + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Kling21MasterArgs { + image_url: image_url, + prompt: "a shot of the mountains, the camera rotates around to show the mountain range, the sun begins to set, and the moon becomes visible", + api_key: &api_key, + duration: Kling21MasterDuration::Default, + aspect_ratio: Kling21MasterAspectRatio::WideSixteenNine, + webhook_url: "https://example.com/webhook", + }; + + let result = enqueue_kling_21_master_image_to_video_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs new file mode 100644 index 0000000000..6fc471584b --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs @@ -0,0 +1,98 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::kling_video::v2_1::pro::image_to_video::{image_to_video, ProImageToVideoRequest}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Kling21ProArgs<'a, U: IntoUrl, V: IntoUrl> { + pub image_url: U, + pub webhook_url: V, + pub prompt: &'a str, + pub api_key: &'a FalApiKey, + pub duration: Kling21ProDuration, + pub aspect_ratio: Kling21ProAspectRatio, +} + +#[derive(Copy, Clone, Debug)] +pub enum Kling21ProDuration { + Default, + FiveSeconds, + TenSeconds, +} + +#[derive(Copy, Clone, Debug)] +pub enum Kling21ProAspectRatio { + Square, // 1:1 + WideSixteenNine, // 16:9 + TallNineSixteen, // 9:16 +} + +pub async fn enqueue_kling_21_pro_image_to_video_webhook( + args: Kling21ProArgs<'_, U, V> +) -> Result { + let duration = match args.duration { + Kling21ProDuration::Default => None, + Kling21ProDuration::FiveSeconds => Some("5".to_string()), // Gross... + Kling21ProDuration::TenSeconds => Some("10".to_string()), + }; + + let aspect_ratio = match args.aspect_ratio { + Kling21ProAspectRatio::Square => Some("1:1".to_string()), + Kling21ProAspectRatio::WideSixteenNine => Some("16:9".to_string()), + Kling21ProAspectRatio::TallNineSixteen => Some("9:16".to_string()), + }; + + let image_url = args.image_url.as_str().to_string(); + + let request = ProImageToVideoRequest { + image_url, + prompt: args.prompt.to_string(), + aspect_ratio, + duration, + // Maybe expose these later + cfg_scale: None, + negative_prompt: None, + tail_image_url: None, + }; + + let result = image_to_video(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::{enqueue_kling_21_pro_image_to_video_webhook, Kling21ProArgs, Kling21ProAspectRatio, Kling21ProDuration}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_kling21_pro_video() -> AnyhowResult<()> { + let image_url = "https://cdn-2.fakeyou.com/media/3/4/h/f/s/34hfsmt8e38rvne6mwa4pwbxr6292sgy/image_34hfsmt8e38rvne6mwa4pwbxr6292sgy.png"; + + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Kling21ProArgs { + image_url: image_url, + prompt: "a shot of the mountains, the camera rotates around to show the mountain range", + api_key: &api_key, + duration: Kling21ProDuration::Default, + aspect_ratio: Kling21ProAspectRatio::WideSixteenNine, + webhook_url: "https://example.com/webhook", + }; + + let result = enqueue_kling_21_pro_image_to_video_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs new file mode 100644 index 0000000000..c13c277dcb --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs @@ -0,0 +1,98 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::bytedance::seedance::v1::lite::image_to_video::image_to_video; +use fal::endpoints::fal_ai::bytedance::seedance::v1::lite::image_to_video::ImageToVideoRequest; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Seedance1LiteArgs<'a, U: IntoUrl, V: IntoUrl> { + pub image_url: U, + pub webhook_url: V, + pub prompt: &'a str, + pub api_key: &'a FalApiKey, + pub camera_fixed: bool, + pub duration: Seedance1LiteDuration, + pub resolution: Seedance1LiteResolution, + pub seed: Option, +} + +#[derive(Copy, Clone, Debug)] +pub enum Seedance1LiteDuration { + FiveSeconds, + TenSeconds, +} + +#[derive(Copy, Clone, Debug)] +pub enum Seedance1LiteResolution { + FourEightyP, // 480p + SevenTwentyP, // 720p +} + +pub async fn enqueue_seedance_1_lite_image_to_video_webhook( + args: Seedance1LiteArgs<'_, U, V> +) -> Result { + let duration = match args.duration { + Seedance1LiteDuration::FiveSeconds => Some("5".to_string()), + Seedance1LiteDuration::TenSeconds => Some("10".to_string()), + }; + + let resolution = match args.resolution { + Seedance1LiteResolution::FourEightyP => Some("480p".to_string()), + Seedance1LiteResolution::SevenTwentyP => Some("720p".to_string()), + }; + + let image_url = args.image_url.as_str().to_string(); + + let request = ImageToVideoRequest { + image_url, + prompt: args.prompt.to_string(), + duration, + resolution, + // TODO: Add these later + camera_fixed: None, + seed: -1, + }; + + let result = image_to_video(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::video::enqueue_seedance_1_lite_image_to_video_webhook::{enqueue_seedance_1_lite_image_to_video_webhook, Seedance1LiteArgs, Seedance1LiteDuration, Seedance1LiteResolution}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_kling21_pro_video() -> AnyhowResult<()> { + let image_url = "https://cdn-2.fakeyou.com/media/3/4/h/f/s/34hfsmt8e38rvne6mwa4pwbxr6292sgy/image_34hfsmt8e38rvne6mwa4pwbxr6292sgy.png"; + + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Seedance1LiteArgs { + image_url: image_url, + prompt: "a shot of the mountains as the sun sets and reveals the moon and stars", + api_key: &api_key, + camera_fixed: false, + duration: Seedance1LiteDuration::FiveSeconds, + resolution: Seedance1LiteResolution::SevenTwentyP, + seed: None, + webhook_url: "https://example.com/webhook", + }; + + let result = enqueue_seedance_1_lite_image_to_video_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_veo_2_image_to_video_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_veo_2_image_to_video_webhook.rs new file mode 100644 index 0000000000..08398556cf --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_veo_2_image_to_video_webhook.rs @@ -0,0 +1,102 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::veo2::image_to_video::image_to_video; +use fal::endpoints::fal_ai::veo2::image_to_video::ImageToVideoInput; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Veo2Args<'a, U: IntoUrl, V: IntoUrl> { + pub image_url: U, + pub webhook_url: V, + pub prompt: &'a str, + pub api_key: &'a FalApiKey, + pub duration: Veo2Duration, + pub aspect_ratio: Veo2AspectRatio, +} + +#[derive(Copy, Clone, Debug)] +pub enum Veo2Duration { + Default, + FiveSeconds, + SixSeconds, + SevenSeconds, + EightSeconds, +} + +#[derive(Copy, Clone, Debug)] +pub enum Veo2AspectRatio { + Auto, + AutoPreferPortrait, + WideSixteenNine, // 16:9 + TallNineSixteen, // 9:16 +} + +pub async fn enqueue_veo_2_image_to_video_webhook( + args: Veo2Args<'_, U, V> +) -> Result { + let duration = match args.duration { + Veo2Duration::Default => None, + Veo2Duration::FiveSeconds => Some("5".to_string()), + Veo2Duration::SixSeconds => Some("6".to_string()), + Veo2Duration::SevenSeconds => Some("7".to_string()), + Veo2Duration::EightSeconds => Some("8".to_string()), + }; + + let aspect_ratio = match args.aspect_ratio { + Veo2AspectRatio::Auto => Some("auto".to_string()), + Veo2AspectRatio::AutoPreferPortrait => Some("auto_prefer_portrait".to_string()), + Veo2AspectRatio::WideSixteenNine => Some("16:9".to_string()), + Veo2AspectRatio::TallNineSixteen => Some("9:16".to_string()), + }; + + let image_url = args.image_url.as_str().to_string(); + + let request = ImageToVideoInput { + image_url, + prompt: args.prompt.to_string(), + aspect_ratio, + duration, + // Maybe expose these later + }; + + let result = image_to_video(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::video::enqueue_veo_2_image_to_video_webhook::{enqueue_veo_2_image_to_video_webhook, Veo2Args, Veo2AspectRatio, Veo2Duration}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_kling21_pro_video() -> AnyhowResult<()> { + let image_url = "https://cdn-2.fakeyou.com/media/3/4/h/f/s/34hfsmt8e38rvne6mwa4pwbxr6292sgy/image_34hfsmt8e38rvne6mwa4pwbxr6292sgy.png"; + + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Veo2Args { + image_url: image_url, + prompt: "a shot of the mountains as the sun sets and reveals the moon and stars", + api_key: &api_key, + duration: Veo2Duration::Default, + aspect_ratio: Veo2AspectRatio::WideSixteenNine, + webhook_url: "https://example.com/webhook", + }; + + let result = enqueue_veo_2_image_to_video_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/video/mod.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/video/mod.rs index 74dead225d..73dd86fd6b 100644 --- a/crates/lib/api_clients/fal_client/src/requests/webhook/video/mod.rs +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/video/mod.rs @@ -1 +1,5 @@ -pub mod enqueue_kling_16_image_to_video_webhook; \ No newline at end of file +pub mod enqueue_kling_16_pro_image_to_video_webhook; +pub mod enqueue_kling_21_master_image_to_video_webhook; +pub mod enqueue_kling_21_pro_image_to_video_webhook; +pub mod enqueue_seedance_1_lite_image_to_video_webhook; +pub mod enqueue_veo_2_image_to_video_webhook; \ No newline at end of file diff --git a/crates/lib/api_clients/storyteller_client/src/generate/image/generate_flux_pro_11_ultra_text_to_image.rs b/crates/lib/api_clients/storyteller_client/src/generate/image/generate_flux_pro_11_ultra_text_to_image.rs new file mode 100644 index 0000000000..dc8a25ad99 --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/generate/image/generate_flux_pro_11_ultra_text_to_image.rs @@ -0,0 +1,68 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::api_error::ApiError; +use crate::error::client_error::ClientError; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::constants::{APPLICATION_JSON, USER_AGENT}; +use crate::utils::filter_bad_response::filter_bad_response; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::{GenerateFluxPro11UltraTextToImageRequest, GenerateFluxPro11UltraTextToImageResponse, GENERATE_FLUX_PRO_11_ULTRA_TEXT_TO_IMAGE_PATH}; +use log::{debug, info}; +use reqwest::Client; + +pub async fn generate_flux_pro_11_ultra_text_to_image( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: GenerateFluxPro11UltraTextToImageRequest, +) -> Result { + + let url = get_route(api_host); + + info!("Requesting {:?}", &url); + + let client = Client::builder() + .gzip(true) + .build() + .map_err(|err| StorytellerError::Client(ClientError::from(err)))?; + + let mut request_builder = client.post(url) + .header("User-Agent", USER_AGENT) + .header("Accept", APPLICATION_JSON) + .header("Content-Type", APPLICATION_JSON); + + if let Some(creds) = maybe_creds { + if let Some(header) = &creds.maybe_as_cookie_header() { + request_builder = request_builder.header("Cookie", header); + } + } + + let request_body = serde_json::to_string(&request) + .map_err(|err| StorytellerError::Client(ClientError::from(err)))?; + + debug!("Request body: {:?}", request_body); + + let response = request_builder + .body(request_body) + .send() + .await + .map_err(|err| StorytellerError::Api(ApiError::from(err)))?; + + let response = filter_bad_response(response) + .await + .map_err(|err| StorytellerError::Api(ApiError::from(err)))?; + + let response_body = &response.text() + .await + .map_err(|err| StorytellerError::Api(ApiError::from(err)))?; + + debug!("Response body: {:?}", response_body); + + let response = serde_json::from_str(&response_body) + .map_err(|err| StorytellerError::Api(ApiError::from(err)))?; + + Ok(response) +} + +fn get_route(api_host: &ApiHost) -> String { + let api_hostname = api_host.to_api_hostname(); + format!("https://{}{}", api_hostname, GENERATE_FLUX_PRO_11_ULTRA_TEXT_TO_IMAGE_PATH) +} diff --git a/crates/lib/api_clients/storyteller_client/src/generate/image/mod.rs b/crates/lib/api_clients/storyteller_client/src/generate/image/mod.rs new file mode 100644 index 0000000000..c7b7937d3a --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/generate/image/mod.rs @@ -0,0 +1 @@ +pub mod generate_flux_pro_11_ultra_text_to_image; \ No newline at end of file diff --git a/crates/lib/api_clients/storyteller_client/src/generate/mod.rs b/crates/lib/api_clients/storyteller_client/src/generate/mod.rs index 305645c9d4..c70c64e67d 100644 --- a/crates/lib/api_clients/storyteller_client/src/generate/mod.rs +++ b/crates/lib/api_clients/storyteller_client/src/generate/mod.rs @@ -1 +1,3 @@ -pub mod object; \ No newline at end of file +pub mod image; +pub mod object; +pub mod video; \ No newline at end of file diff --git a/crates/lib/api_clients/storyteller_client/src/generate/object/generate_hunyuan2_image_to_3d.rs b/crates/lib/api_clients/storyteller_client/src/generate/object/generate_hunyuan2_image_to_3d.rs index 789efa1ebb..48764f1ecc 100644 --- a/crates/lib/api_clients/storyteller_client/src/generate/object/generate_hunyuan2_image_to_3d.rs +++ b/crates/lib/api_clients/storyteller_client/src/generate/object/generate_hunyuan2_image_to_3d.rs @@ -9,7 +9,6 @@ use artcraft_api_defs::generate::object::generate_hunyuan_2_image_to_3d::{Genera use log::{debug, info}; use reqwest::Client; -/// Upload an image media file from a file. pub async fn generate_hunyuan2_image_to_3d( api_host: &ApiHost, maybe_creds: Option<&StorytellerCredentialSet>, diff --git a/crates/lib/api_clients/storyteller_client/src/generate/video/generate_kling_16_pro_image_to_video.rs b/crates/lib/api_clients/storyteller_client/src/generate/video/generate_kling_16_pro_image_to_video.rs new file mode 100644 index 0000000000..c3164adf65 --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/generate/video/generate_kling_16_pro_image_to_video.rs @@ -0,0 +1,68 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::api_error::ApiError; +use crate::error::client_error::ClientError; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::constants::{APPLICATION_JSON, USER_AGENT}; +use crate::utils::filter_bad_response::filter_bad_response; +use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{GenerateKling16ProImageToVideoRequest, GenerateKling16ProImageToVideoResponse, GENERATE_KLING_1_6_PRO_IMAGE_TO_VIDEO_URL_PATH}; +use log::{debug, info}; +use reqwest::Client; + +pub async fn generate_kling_16_pro_image_to_video( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: GenerateKling16ProImageToVideoRequest, +) -> Result { + + let url = get_route(api_host); + + info!("Requesting {:?}", &url); + + let client = Client::builder() + .gzip(true) + .build() + .map_err(|err| StorytellerError::Client(ClientError::from(err)))?; + + let mut request_builder = client.post(url) + .header("User-Agent", USER_AGENT) + .header("Accept", APPLICATION_JSON) + .header("Content-Type", APPLICATION_JSON); + + if let Some(creds) = maybe_creds { + if let Some(header) = &creds.maybe_as_cookie_header() { + request_builder = request_builder.header("Cookie", header); + } + } + + let request_body = serde_json::to_string(&request) + .map_err(|err| StorytellerError::Client(ClientError::from(err)))?; + + debug!("Request body: {:?}", request_body); + + let response = request_builder + .body(request_body) + .send() + .await + .map_err(|err| StorytellerError::Api(ApiError::from(err)))?; + + let response = filter_bad_response(response) + .await + .map_err(|err| StorytellerError::Api(ApiError::from(err)))?; + + let response_body = &response.text() + .await + .map_err(|err| StorytellerError::Api(ApiError::from(err)))?; + + debug!("Response body: {:?}", response_body); + + let response = serde_json::from_str(&response_body) + .map_err(|err| StorytellerError::Api(ApiError::from(err)))?; + + Ok(response) +} + +fn get_route(api_host: &ApiHost) -> String { + let api_hostname = api_host.to_api_hostname(); + format!("https://{}{}", api_hostname, GENERATE_KLING_1_6_PRO_IMAGE_TO_VIDEO_URL_PATH) +} diff --git a/crates/lib/api_clients/storyteller_client/src/generate/video/mod.rs b/crates/lib/api_clients/storyteller_client/src/generate/video/mod.rs new file mode 100644 index 0000000000..f2afa94ca9 --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/generate/video/mod.rs @@ -0,0 +1 @@ +pub mod generate_kling_16_pro_image_to_video; \ No newline at end of file diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/generate_flux_pro_11_ultra_text_to_image.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/generate_flux_pro_11_ultra_text_to_image.rs new file mode 100644 index 0000000000..e1a2a7de8d --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/generate_flux_pro_11_ultra_text_to_image.rs @@ -0,0 +1,20 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use utoipa::ToSchema; + +pub const GENERATE_FLUX_PRO_11_ULTRA_TEXT_TO_IMAGE_PATH: &str = "/v1/generate/image/flux_pro_1.1_ultra_text_to_image"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateFluxPro11UltraTextToImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Text prompt to generate the image from. + pub prompt: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateFluxPro11UltraTextToImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs index e9f3da729c..ed74641a70 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs @@ -1 +1,2 @@ +pub mod generate_flux_pro_11_ultra_text_to_image; pub mod remove_image_background; \ No newline at end of file diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_pro_image_to_video.rs similarity index 53% rename from crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_image_to_video.rs rename to crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_pro_image_to_video.rs index fc85085a03..2a03c28862 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_image_to_video.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_pro_image_to_video.rs @@ -3,10 +3,10 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; use utoipa::ToSchema; -const GENERATE_KLING_1_6_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/kling_16_image_to_video"; +pub const GENERATE_KLING_1_6_PRO_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/kling_1.6_pro_image_to_video"; #[derive(Serialize, Deserialize, ToSchema)] -pub struct GenerateKling16ImageToVideoRequest { +pub struct GenerateKling16ProImageToVideoRequest { /// Idempotency token to prevent duplicate requests. pub uuid_idempotency_token: String, @@ -16,10 +16,23 @@ pub struct GenerateKling16ImageToVideoRequest { /// Optional text prompt. pub prompt: Option, + + /// Optional: apsect ratio of the generated video. + pub aspect_ratio: Option, } #[derive(Serialize, Deserialize, ToSchema)] -pub struct GenerateKling16ImageToVideoResponse { +pub enum GenerateKling16ProAspectRatio { + /// 16:9 aspect ratio + WideSixteenNine, + /// 9:16 aspect ratio + TallNineSixteen, + /// 1:1 aspect ratio + Square, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateKling16ProImageToVideoResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs index 2c1da8e82a..5a1956a8f5 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs @@ -1 +1 @@ -pub mod generate_kling_1_6_image_to_video; \ No newline at end of file +pub mod generate_kling_1_6_pro_image_to_video; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/generate_flux_pro_ultra_text_to_image_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/generate_flux_pro_ultra_text_to_image_handler.rs new file mode 100644 index 0000000000..c951d688ff --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/generate_flux_pro_ultra_text_to_image_handler.rs @@ -0,0 +1,175 @@ +use std::fmt; +use std::sync::Arc; + +use crate::http_server::common_requests::media_file_token_path_info::MediaFileTokenPathInfo; +use crate::http_server::common_responses::media::media_links::MediaLinks; +use crate::http_server::deprecated_endpoints::engine::create_scene_handler::CreateSceneError; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::state::server_state::ServerState; +use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Json; +use actix_web::web::Path; +use actix_web::{web, HttpRequest, HttpResponse}; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageResponse; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::common::visibility::Visibility; +use fal_client::requests::webhook::image::enqueue_flux_pro_11_ultra_text_to_image_webhook::enqueue_flux_pro_11_ultra_text_to_image_webhook; +use fal_client::requests::webhook::image::enqueue_flux_pro_11_ultra_text_to_image_webhook::FluxPro11UltraArgs; +use http_server_common::request::get_request_ip::get_request_ip; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use idempotency::uuid::generate_random_uuid; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, MediaFile}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +// =============== Error Response =============== + + +#[derive(Debug, Serialize, ToSchema)] +pub enum GenerateFluxProUltraTextToImageError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for GenerateFluxProUltraTextToImageError { + fn status_code(&self) -> StatusCode { + match *self { + GenerateFluxProUltraTextToImageError::BadInput(_) => StatusCode::BAD_REQUEST, + GenerateFluxProUltraTextToImageError::NotFound => StatusCode::NOT_FOUND, + GenerateFluxProUltraTextToImageError::NotAuthorized => StatusCode::UNAUTHORIZED, + GenerateFluxProUltraTextToImageError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GenerateFluxProUltraTextToImageError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +/// Flux pro ultra text to image +#[utoipa::path( + post, + tag = "Generate Images", + path = "/v1/generate/image/flux_pro_1.1_ultra_text_to_image", + responses( + (status = 200, description = "Success", body = GenerateFluxProUltraTextToImageResponse), + (status = 400, description = "Bad input", body = GenerateFluxProUltraTextToImageError), + (status = 401, description = "Not authorized", body = GenerateFluxProUltraTextToImageError), + (status = 500, description = "Server error", body = GenerateFluxProUltraTextToImageError), + ), + params( + ("request" = GenerateFluxProUltraTextToImageRequest, description = "Payload for Request"), + ) +)] +pub async fn generate_flux_pro_11_ultra_text_to_image_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, GenerateFluxProUltraTextToImageError> { + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GenerateFluxProUltraTextToImageError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(GenerateFluxProUltraTextToImageError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(GenerateFluxProUltraTextToImageError::BadInput(reason)); + } + + insert_idempotency_token(&request.uuid_idempotency_token, &server_state.mysql_pool) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + GenerateFluxProUltraTextToImageError::BadInput("invalid idempotency token".to_string()) + })?; + const IS_MOD : bool = false; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let args = FluxPro11UltraArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + let fal_result = enqueue_flux_pro_11_ultra_text_to_image_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_flux_pro_ultra_text_to_image_webhook: {:?}", err); + GenerateFluxProUltraTextToImageError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + GenerateFluxProUltraTextToImageError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_pool: &server_state.mysql_pool, + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(GenerateFluxProUltraTextToImageError::ServerError); + } + }; + + Ok(Json(GenerateFluxPro11UltraTextToImageResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs index 866ad9811a..48b150f9f1 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs @@ -1 +1,2 @@ +pub mod generate_flux_pro_ultra_text_to_image_handler; pub mod remove_image_background_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/object/generate_hunyuan2_image_to_3d_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/object/generate_hunyuan2_image_to_3d_handler.rs index cca36b1b8a..14a824ae92 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/object/generate_hunyuan2_image_to_3d_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/object/generate_hunyuan2_image_to_3d_handler.rs @@ -19,8 +19,8 @@ use artcraft_api_defs::generate::object::generate_hunyuan_2_image_to_3d::Generat use artcraft_api_defs::generate::object::generate_hunyuan_2_image_to_3d::GenerateHunyuan2ImageTo3dResponse; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::common::visibility::Visibility; -use fal_client::requests::webhook::object::enqueue_hunyuan2_image_to_3d_webhook::enqueue_hunyuan2_image_to_3d_webhook; -use fal_client::requests::webhook::object::enqueue_hunyuan2_image_to_3d_webhook::Hunyuan2Args; +use fal_client::requests::webhook::object::enqueue_hunyuan_3d_2_image_to_3d_webhook::enqueue_hunyuan_3d_2_image_to_3d_webhook; +use fal_client::requests::webhook::object::enqueue_hunyuan_3d_2_image_to_3d_webhook::Hunyuan3d2Args; use http_server_common::request::get_request_ip::get_request_ip; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use idempotency::uuid::generate_random_uuid; @@ -167,13 +167,13 @@ pub async fn generate_hunyuan_2_image_to_3d_handler( info!("Fal webhook URL: {}", server_state.fal.webhook_url); - let args = Hunyuan2Args { + let args = Hunyuan3d2Args { image_url: media_links.cdn_url, webhook_url: &server_state.fal.webhook_url, api_key: &server_state.fal.api_key, }; - let fal_result = enqueue_hunyuan2_image_to_3d_webhook(args) + let fal_result = enqueue_hunyuan_3d_2_image_to_3d_webhook(args) .await .map_err(|err| { warn!("Error calling enqueue_hunyuan2_image_to_3d_webhook: {:?}", err); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_pro_video_handler.rs similarity index 79% rename from crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_video_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_pro_video_handler.rs index 4b38556e7f..5fb024db90 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_video_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_pro_video_handler.rs @@ -15,13 +15,14 @@ use actix_web::http::StatusCode; use actix_web::web::Json; use actix_web::web::Path; use actix_web::{web, HttpRequest, HttpResponse}; -use artcraft_api_defs::generate::video::generate_kling_1_6_image_to_video::GenerateKling16ImageToVideoRequest; -use artcraft_api_defs::generate::video::generate_kling_1_6_image_to_video::GenerateKling16ImageToVideoResponse; +use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::GenerateKling16ProImageToVideoResponse; +use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{GenerateKling16ProAspectRatio, GenerateKling16ProImageToVideoRequest}; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::common::visibility::Visibility; -use fal_client::requests::webhook::video::enqueue_kling_16_image_to_video_webhook::enqueue_kling_16_image_to_video_webhook; -use fal_client::requests::webhook::video::enqueue_kling_16_image_to_video_webhook::Kling16Args; -use fal_client::requests::webhook::video::enqueue_kling_16_image_to_video_webhook::Kling16Duration; +use fal_client::requests::webhook::video::enqueue_kling_16_pro_image_to_video_webhook::enqueue_kling_16_pro_image_to_video_webhook; +use fal_client::requests::webhook::video::enqueue_kling_16_pro_image_to_video_webhook::Kling16Duration; +use fal_client::requests::webhook::video::enqueue_kling_16_pro_image_to_video_webhook::Kling16ProArgs; +use fal_client::requests::webhook::video::enqueue_kling_16_pro_image_to_video_webhook::Kling16ProAspectRatio; use http_server_common::request::get_request_ip::get_request_ip; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use idempotency::uuid::generate_random_uuid; @@ -37,6 +38,29 @@ use utoipa::ToSchema; // =============== Error Response =============== +/* + TODO: Either on the Tauri client or as an API, make "generic" image / video / object endpoints. + Take a reference image payload: + + ReferenceImage { + media_token: MediaFileToken, + reference_type: ReferenceType + } + + enum ReferenceType { + PrimaryReference, + CharacterReference, + LocationReference, + StyleReference, + ... + GenericReference, + } + + // **ORDER MATTERS** + reference_images: Vec, + +*/ + #[derive(Debug, Serialize, ToSchema)] pub enum GenerateKling16VideoError { BadInput(String), @@ -73,7 +97,7 @@ impl fmt::Display for GenerateKling16VideoError { #[utoipa::path( post, tag = "Generate Videos", - path = "/v1/generate/video/kling_16_image_to_video", + path = "/v1/generate/video/kling_1.6_pro_image_to_video", responses( (status = 200, description = "Success", body = RemoveImageBackgroundResponse), (status = 400, description = "Bad input", body = RemoveImageBackgroundError), @@ -84,11 +108,11 @@ impl fmt::Display for GenerateKling16VideoError { ("request" = RemoveImageBackgroundRequest, description = "Payload for Request"), ) )] -pub async fn generate_kling_1_6_video_handler( +pub async fn generate_kling_1_6_pro_video_handler( http_request: HttpRequest, - request: Json, + request: Json, server_state: web::Data> -) -> Result, GenerateKling16VideoError> { +) -> Result, GenerateKling16VideoError> { let maybe_user_session = server_state .session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) @@ -173,15 +197,23 @@ pub async fn generate_kling_1_6_video_handler( .map(|prompt| prompt.trim()) .unwrap_or_else(|| ""); - let args = Kling16Args { + let aspect_ratio = match &request.aspect_ratio { + Some(GenerateKling16ProAspectRatio::Square) => Kling16ProAspectRatio::Square, + Some(GenerateKling16ProAspectRatio::WideSixteenNine) => Kling16ProAspectRatio::WideSixteenNine, + Some(GenerateKling16ProAspectRatio::TallNineSixteen) => Kling16ProAspectRatio::TallNineSixteen, + None => Kling16ProAspectRatio::WideSixteenNine, // Default to 16:9 + }; + + let args = Kling16ProArgs { image_url: media_links.cdn_url, webhook_url: &server_state.fal.webhook_url, duration: Kling16Duration::Default, prompt: prompt, + aspect_ratio: aspect_ratio, api_key: &server_state.fal.api_key, }; - let fal_result = enqueue_kling_16_image_to_video_webhook(args) + let fal_result = enqueue_kling_16_pro_image_to_video_webhook(args) .await .map_err(|err| { warn!("Error calling remove_background_rembg_webhook: {:?}", err); @@ -218,7 +250,7 @@ pub async fn generate_kling_1_6_video_handler( } }; - Ok(Json(GenerateKling16ImageToVideoResponse { + Ok(Json(GenerateKling16ProImageToVideoResponse { success: true, inference_job_token: job_token, })) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs index 272055b52e..d7a0e16804 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs @@ -1 +1 @@ -pub mod generate_kling_1_6_video_handler; \ No newline at end of file +pub mod generate_kling_1_6_pro_video_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs index f3b1ada7bd..4ab6f6bbcc 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs @@ -1,6 +1,7 @@ +use crate::http_server::endpoints::generate::image::generate_flux_pro_ultra_text_to_image_handler::generate_flux_pro_11_ultra_text_to_image_handler; use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler; use crate::http_server::endpoints::generate::object::generate_hunyuan2_image_to_3d_handler::generate_hunyuan_2_image_to_3d_handler; -use crate::http_server::endpoints::generate::video::generate_kling_1_6_video_handler::generate_kling_1_6_video_handler; +use crate::http_server::endpoints::generate::video::generate_kling_1_6_pro_video_handler::generate_kling_1_6_pro_video_handler; use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; @@ -19,14 +20,18 @@ where { app.service(web::scope("/v1/generate") .service(web::scope("/image") + .service(web::resource("/flux_pro_1.1_ultra_text_to_image") + .route(web::post().to(generate_flux_pro_11_ultra_text_to_image_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/remove_background") .route(web::post().to(remove_image_background_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) ) .service(web::scope("/video") - .service(web::resource("/kling_16_image_to_video") - .route(web::post().to(generate_kling_1_6_video_handler)) + .service(web::resource("/kling_1.6_pro_image_to_video") + .route(web::post().to(generate_kling_1_6_pro_video_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) ) diff --git a/crates/vendor/fal-rs/Cargo.toml b/crates/vendor/fal-rs/Cargo.toml index fe3dd4f147..f22963f498 100644 --- a/crates/vendor/fal-rs/Cargo.toml +++ b/crates/vendor/fal-rs/Cargo.toml @@ -86,6 +86,7 @@ endpoints_fal-ai_auto-caption = [] endpoints_fal-ai_ben = [] endpoints_fal-ai_birefnet = [] endpoints_fal-ai_bria = [] +endpoints_fal-ai_bytedance = [] endpoints_fal-ai_cat-vton = [] endpoints_fal-ai_ccsr = [] endpoints_fal-ai_clarity-upscaler = [] diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/mod.rs new file mode 100644 index 0000000000..ef75917b3f --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/mod.rs @@ -0,0 +1,14 @@ +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" + ))) +)] +pub mod seedance; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/mod.rs new file mode 100644 index 0000000000..3c690b3985 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/mod.rs @@ -0,0 +1,14 @@ +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" + ))) +)] +pub mod v1; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs new file mode 100644 index 0000000000..017f2d7f39 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs @@ -0,0 +1,69 @@ +#[allow(unused_imports)] +use crate::prelude::*; +#[allow(unused_imports)] +use serde::{Deserialize, Serialize}; +#[allow(unused_imports)] +use std::collections::HashMap; + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct File { + /// The mime type of the file. + /// "image/png" + #[serde(skip_serializing_if = "Option::is_none")] + pub content_type: Option, + /// File data + #[serde(skip_serializing_if = "Option::is_none")] + pub file_data: Option, + /// The name of the file. It will be auto-generated if not provided. + /// "z9RV14K95DvU.png" + #[serde(skip_serializing_if = "Option::is_none")] + pub file_name: Option, + /// The size of the file in bytes. + /// 4404019 + #[serde(skip_serializing_if = "Option::is_none")] + pub file_size: Option, + /// The URL where the file can be downloaded from. + pub url: String, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct HTTPValidationError { + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option>>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct I2VOutput { + /// The generated video + /// {"url":"https://v2.fal.media/files/36087878b0c1435bb75c19b64b7db178_output.mp4"} + pub video: File, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ImageToVideoRequest { + /// The resolution of the generated video frame + #[serde(skip_serializing_if = "Option::is_none")] + pub resolution: Option, + /// The duration of the generated video in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + pub image_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub camera_fixed: Option, + pub prompt: String, + pub seed: i32, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ValidationError { + pub loc: Vec, + pub msg: String, + #[serde(rename = "type")] + pub ty: String, +} + +pub fn image_to_video( + params: ImageToVideoRequest, +) -> FalRequest { + FalRequest::new("fal-ai/bytedance/seedance/v1/lite/image-to-video", params) +} diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/mod.rs new file mode 100644 index 0000000000..6de2bbb953 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/mod.rs @@ -0,0 +1,14 @@ +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" + ))) +)] +pub mod image_to_video; diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/mod.rs new file mode 100644 index 0000000000..6cbbc2be2a --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/mod.rs @@ -0,0 +1,14 @@ +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" + ))) +)] +pub mod lite; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/hunyuan3d/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/hunyuan3d/mod.rs index 8eadf1faa1..0d46dc31f0 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/hunyuan3d/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/hunyuan3d/mod.rs @@ -12,3 +12,17 @@ ))) )] pub mod v2; +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_hunyuan3d" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_hunyuan3d" + ))) +)] +pub mod v21; diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/hunyuan3d/v21/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/hunyuan3d/v21/mod.rs new file mode 100644 index 0000000000..80e703521a --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/hunyuan3d/v21/mod.rs @@ -0,0 +1,64 @@ +use crate::prelude::FalRequest; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Hunyuan3DInput { + /// Guidance scale for the model. + #[serde(skip_serializing_if = "Option::is_none")] + pub guidance_scale: Option, + /// URL of image to use while generating the 3D model. + /// "https://storage.googleapis.com/falserverless/model_tests/video_models/robot.png" + pub input_image_url: String, + /// Number of inference steps to perform. + #[serde(skip_serializing_if = "Option::is_none")] + pub num_inference_steps: Option, + /// Octree resolution for the model. + #[serde(skip_serializing_if = "Option::is_none")] + pub octree_resolution: Option, + /// The same seed and the same prompt given to the same version of the model + /// will output the same image every time. + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, + /// If set true, textured mesh will be generated and the price charged would be 3 times that of white mesh. + #[serde(skip_serializing_if = "Option::is_none")] + pub textured_mesh: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ObjectOutput { + /// Generated 3D object file. + /// {"content_type":"application/octet-stream","file_name":"white_mesh.glb","file_size":720696,"url":"https://v3.fal.media/files/lion/WqIhtKPaSoeBtC30qzIGG_white_mesh.glb"} + pub model_mesh: File, + /// Seed value used for generation. + pub seed: i64, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct File { + /// The mime type of the file. + /// "image/png" + #[serde(skip_serializing_if = "Option::is_none")] + pub content_type: Option, + /// File data + #[serde(skip_serializing_if = "Option::is_none")] + pub file_data: Option, + /// The name of the file. It will be auto-generated if not provided. + /// "z9RV14K95DvU.png" + #[serde(skip_serializing_if = "Option::is_none")] + pub file_name: Option, + /// The size of the file in bytes. + /// 4404019 + #[serde(skip_serializing_if = "Option::is_none")] + pub file_size: Option, + /// The URL where the file can be downloaded from. + pub url: String, +} + +/// Hunyuan3D +/// +/// Category: image-to-3d +/// Machine Type: H100 +/// License Type: commercial +pub fn v21(params: Hunyuan3DInput) -> FalRequest { + FalRequest::new("fal-ai/hunyuan3d-v21", params) +} diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/mod.rs index e7b7328051..56863d4604 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/mod.rs @@ -40,3 +40,17 @@ pub mod v1_5; ))) )] pub mod v1_6; +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" + ))) +)] +pub mod v2_1; diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/master/image_to_video/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/master/image_to_video/mod.rs new file mode 100644 index 0000000000..0711f8898c --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/master/image_to_video/mod.rs @@ -0,0 +1,241 @@ +#[allow(unused_imports)] +use crate::prelude::*; +#[allow(unused_imports)] +use serde::{Deserialize, Serialize}; +#[allow(unused_imports)] +use std::collections::HashMap; + +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct CameraControl { +// /// The type of camera movement +// /// "horizontal" +// pub movement_type: String, +// /// The value of the camera movement +// /// 10 +// pub movement_value: i64, +// } + +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct DynamicMask { +// /// URL of the image for Dynamic Brush Application Area (Mask image created by users using the motion brush) +// /// "https://h2.inkwai.com/bs2/upload-ylab-stunt/ai_portal/1732888130/WU8spl23dA/dynamic_mask_1.png" +// pub mask_url: String, +// /// List of trajectories +// /// [{"x":279,"y":219},{"x":417,"y":65}] +// #[serde(skip_serializing_if = "Option::is_none")] +// pub trajectories: Option>>, +// } + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct File { + /// The mime type of the file. + /// "image/png" + #[serde(skip_serializing_if = "Option::is_none")] + pub content_type: Option, + /// File data + #[serde(skip_serializing_if = "Option::is_none")] + pub file_data: Option, + /// The name of the file. It will be auto-generated if not provided. + /// "z9RV14K95DvU.png" + #[serde(skip_serializing_if = "Option::is_none")] + pub file_name: Option, + /// The size of the file in bytes. + /// 4404019 + #[serde(skip_serializing_if = "Option::is_none")] + pub file_size: Option, + /// The URL where the file can be downloaded from. + pub url: String, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct HTTPValidationError { + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option>>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct I2VOutput { + /// The generated video + /// {"url":"https://v2.fal.media/files/36087878b0c1435bb75c19b64b7db178_output.mp4"} + pub video: File, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ImageToVideoRequest { + /// The aspect ratio of the generated video frame + #[serde(skip_serializing_if = "Option::is_none")] + pub aspect_ratio: Option, + /// The CFG (Classifier Free Guidance) scale is a measure of how close you want + /// the model to stick to your prompt. + #[serde(skip_serializing_if = "Option::is_none")] + pub cfg_scale: Option, + /// The duration of the generated video in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + pub image_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub negative_prompt: Option, + pub prompt: String, +} + +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct KlingV1I2VOutput { +// /// The generated video +// /// {"url":"https://v3.fal.media/files/kangaroo/VUmAU3JvzS7mxwdgSU9zj_output.mp4"} +// pub video: File, +// } + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct MasterImageToVideoRequest { + /// The aspect ratio of the generated video frame + #[serde(skip_serializing_if = "Option::is_none")] + pub aspect_ratio: Option, + /// The CFG (Classifier Free Guidance) scale is a measure of how close you want + /// the model to stick to your prompt. + #[serde(skip_serializing_if = "Option::is_none")] + pub cfg_scale: Option, + /// The duration of the generated video in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + pub image_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub negative_prompt: Option, + pub prompt: String, + /// URL of the image to be used for the end of the video + #[serde(skip_serializing_if = "Option::is_none")] + pub tail_image_url: Option, +} + +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct T2VOutput { +// /// The generated video +// /// {"url":"https://v2.fal.media/files/fb33a862b94d4d7195e610e4cbc5d392_output.mp4"} +// pub video: File, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct TextToVideoRequest { +// /// The aspect ratio of the generated video frame +// #[serde(skip_serializing_if = "Option::is_none")] +// pub aspect_ratio: Option, +// /// The CFG (Classifier Free Guidance) scale is a measure of how close you want +// /// the model to stick to your prompt. +// #[serde(skip_serializing_if = "Option::is_none")] +// pub cfg_scale: Option, +// /// The duration of the generated video in seconds +// #[serde(skip_serializing_if = "Option::is_none")] +// pub duration: Option, +// #[serde(skip_serializing_if = "Option::is_none")] +// pub negative_prompt: Option, +// pub prompt: String, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct Trajectory { +// /// X coordinate of the motion trajectory +// /// 279 +// pub x: i64, +// /// Y coordinate of the motion trajectory +// /// 219 +// pub y: i64, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct V1ImageToVideoRequest { +// /// The aspect ratio of the generated video frame +// #[serde(skip_serializing_if = "Option::is_none")] +// pub aspect_ratio: Option, +// /// The CFG (Classifier Free Guidance) scale is a measure of how close you want +// /// the model to stick to your prompt. +// #[serde(skip_serializing_if = "Option::is_none")] +// pub cfg_scale: Option, +// /// The duration of the generated video in seconds +// #[serde(skip_serializing_if = "Option::is_none")] +// pub duration: Option, +// /// List of dynamic masks +// #[serde(skip_serializing_if = "Option::is_none")] +// pub dynamic_masks: Option>>, +// /// URL of the image to be used for the video +// /// "https://h2.inkwai.com/bs2/upload-ylab-stunt/se/ai_portal_queue_mmu_image_upscale_aiweb/3214b798-e1b4-4b00-b7af-72b5b0417420_raw_image_0.jpg" +// pub image_url: String, +// #[serde(skip_serializing_if = "Option::is_none")] +// pub negative_prompt: Option, +// /// The prompt for the video +// /// "The astronaut stood up and walked away" +// pub prompt: String, +// /// URL of the image for Static Brush Application Area (Mask image created by users using the motion brush) +// /// "https://h2.inkwai.com/bs2/upload-ylab-stunt/ai_portal/1732888177/cOLNrShrSO/static_mask.png" +// #[serde(skip_serializing_if = "Option::is_none")] +// pub static_mask_url: Option, +// /// URL of the image to be used for the end of the video +// #[serde(skip_serializing_if = "Option::is_none")] +// pub tail_image_url: Option, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct V1TextToVideoRequest { +// /// Advanced Camera control parameters +// #[serde(skip_serializing_if = "Option::is_none")] +// pub advanced_camera_control: Option>, +// /// The aspect ratio of the generated video frame +// #[serde(skip_serializing_if = "Option::is_none")] +// pub aspect_ratio: Option, +// /// Camera control parameters +// #[serde(skip_serializing_if = "Option::is_none")] +// pub camera_control: Option, +// /// The CFG (Classifier Free Guidance) scale is a measure of how close you want +// /// the model to stick to your prompt. +// #[serde(skip_serializing_if = "Option::is_none")] +// pub cfg_scale: Option, +// /// The duration of the generated video in seconds +// #[serde(skip_serializing_if = "Option::is_none")] +// pub duration: Option, +// #[serde(skip_serializing_if = "Option::is_none")] +// pub negative_prompt: Option, +// pub prompt: String, +// } +// +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ValidationError { + pub loc: Vec, + pub msg: String, + #[serde(rename = "type")] + pub ty: String, +} +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct VideoEffectsOutput { +// /// The generated video +// /// {"content_type":"video/mp4","file_name":"output.mp4","url":"https://storage.googleapis.com/falserverless/kling/kling_ex.mp4.mp4"} +// pub video: File, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct VideoEffectsRequest { +// /// The duration of the generated video in seconds +// #[serde(skip_serializing_if = "Option::is_none")] +// pub duration: Option, +// /// The effect scene to use for the video generation +// /// "hug" +// pub effect_scene: String, +// /// URL of the image to be used for the squish and expansion video +// /// "https://h2.inkwai.com/bs2/upload-ylab-stunt/se/ai_portal_queue_mmu_image_upscale_aiweb/3214b798-e1b4-4b00-b7af-72b5b0417420_raw_image_0.jpg" +// pub image_url: String, +// /// URL of images to be used for hug, kiss or heart_gesture video. +// /// ["https://storage.googleapis.com/falserverless/juggernaut_examples/VHXMavzPyI27zi6JseyL4.png","https://storage.googleapis.com/falserverless/juggernaut_examples/QEW5VrzccxGva7mPfEXjf.png"] +// #[serde(skip_serializing_if = "Option::is_none")] +// pub input_image_urls: Option>>, +// } + +/// Kling 1.0 +/// +/// Category: text-to-video +/// Machine Type: A100 +/// +/// +/// Kling 1.6 (pro) Image to Video API. +pub fn image_to_video( + params: MasterImageToVideoRequest, +) -> FalRequest { + FalRequest::new("fal-ai/kling-video/v2.1/master/image-to-video", params) +} diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/master/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/master/mod.rs new file mode 100644 index 0000000000..30605e2d08 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/master/mod.rs @@ -0,0 +1,14 @@ +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" + ))) +)] +pub mod image_to_video; diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/mod.rs new file mode 100644 index 0000000000..1e3f4ca7c1 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/mod.rs @@ -0,0 +1,28 @@ +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" + ))) +)] +pub mod master; +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" + ))) +)] +pub mod pro; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/pro/image_to_video/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/pro/image_to_video/mod.rs new file mode 100644 index 0000000000..34bbed2e3b --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/pro/image_to_video/mod.rs @@ -0,0 +1,241 @@ +#[allow(unused_imports)] +use crate::prelude::*; +#[allow(unused_imports)] +use serde::{Deserialize, Serialize}; +#[allow(unused_imports)] +use std::collections::HashMap; + +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct CameraControl { +// /// The type of camera movement +// /// "horizontal" +// pub movement_type: String, +// /// The value of the camera movement +// /// 10 +// pub movement_value: i64, +// } + +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct DynamicMask { +// /// URL of the image for Dynamic Brush Application Area (Mask image created by users using the motion brush) +// /// "https://h2.inkwai.com/bs2/upload-ylab-stunt/ai_portal/1732888130/WU8spl23dA/dynamic_mask_1.png" +// pub mask_url: String, +// /// List of trajectories +// /// [{"x":279,"y":219},{"x":417,"y":65}] +// #[serde(skip_serializing_if = "Option::is_none")] +// pub trajectories: Option>>, +// } + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct File { + /// The mime type of the file. + /// "image/png" + #[serde(skip_serializing_if = "Option::is_none")] + pub content_type: Option, + /// File data + #[serde(skip_serializing_if = "Option::is_none")] + pub file_data: Option, + /// The name of the file. It will be auto-generated if not provided. + /// "z9RV14K95DvU.png" + #[serde(skip_serializing_if = "Option::is_none")] + pub file_name: Option, + /// The size of the file in bytes. + /// 4404019 + #[serde(skip_serializing_if = "Option::is_none")] + pub file_size: Option, + /// The URL where the file can be downloaded from. + pub url: String, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct HTTPValidationError { + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option>>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct I2VOutput { + /// The generated video + /// {"url":"https://v2.fal.media/files/36087878b0c1435bb75c19b64b7db178_output.mp4"} + pub video: File, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ImageToVideoRequest { + /// The aspect ratio of the generated video frame + #[serde(skip_serializing_if = "Option::is_none")] + pub aspect_ratio: Option, + /// The CFG (Classifier Free Guidance) scale is a measure of how close you want + /// the model to stick to your prompt. + #[serde(skip_serializing_if = "Option::is_none")] + pub cfg_scale: Option, + /// The duration of the generated video in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + pub image_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub negative_prompt: Option, + pub prompt: String, +} + +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct KlingV1I2VOutput { +// /// The generated video +// /// {"url":"https://v3.fal.media/files/kangaroo/VUmAU3JvzS7mxwdgSU9zj_output.mp4"} +// pub video: File, +// } + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ProImageToVideoRequest { + /// The aspect ratio of the generated video frame + #[serde(skip_serializing_if = "Option::is_none")] + pub aspect_ratio: Option, + /// The CFG (Classifier Free Guidance) scale is a measure of how close you want + /// the model to stick to your prompt. + #[serde(skip_serializing_if = "Option::is_none")] + pub cfg_scale: Option, + /// The duration of the generated video in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + pub image_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub negative_prompt: Option, + pub prompt: String, + /// URL of the image to be used for the end of the video + #[serde(skip_serializing_if = "Option::is_none")] + pub tail_image_url: Option, +} + +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct T2VOutput { +// /// The generated video +// /// {"url":"https://v2.fal.media/files/fb33a862b94d4d7195e610e4cbc5d392_output.mp4"} +// pub video: File, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct TextToVideoRequest { +// /// The aspect ratio of the generated video frame +// #[serde(skip_serializing_if = "Option::is_none")] +// pub aspect_ratio: Option, +// /// The CFG (Classifier Free Guidance) scale is a measure of how close you want +// /// the model to stick to your prompt. +// #[serde(skip_serializing_if = "Option::is_none")] +// pub cfg_scale: Option, +// /// The duration of the generated video in seconds +// #[serde(skip_serializing_if = "Option::is_none")] +// pub duration: Option, +// #[serde(skip_serializing_if = "Option::is_none")] +// pub negative_prompt: Option, +// pub prompt: String, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct Trajectory { +// /// X coordinate of the motion trajectory +// /// 279 +// pub x: i64, +// /// Y coordinate of the motion trajectory +// /// 219 +// pub y: i64, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct V1ImageToVideoRequest { +// /// The aspect ratio of the generated video frame +// #[serde(skip_serializing_if = "Option::is_none")] +// pub aspect_ratio: Option, +// /// The CFG (Classifier Free Guidance) scale is a measure of how close you want +// /// the model to stick to your prompt. +// #[serde(skip_serializing_if = "Option::is_none")] +// pub cfg_scale: Option, +// /// The duration of the generated video in seconds +// #[serde(skip_serializing_if = "Option::is_none")] +// pub duration: Option, +// /// List of dynamic masks +// #[serde(skip_serializing_if = "Option::is_none")] +// pub dynamic_masks: Option>>, +// /// URL of the image to be used for the video +// /// "https://h2.inkwai.com/bs2/upload-ylab-stunt/se/ai_portal_queue_mmu_image_upscale_aiweb/3214b798-e1b4-4b00-b7af-72b5b0417420_raw_image_0.jpg" +// pub image_url: String, +// #[serde(skip_serializing_if = "Option::is_none")] +// pub negative_prompt: Option, +// /// The prompt for the video +// /// "The astronaut stood up and walked away" +// pub prompt: String, +// /// URL of the image for Static Brush Application Area (Mask image created by users using the motion brush) +// /// "https://h2.inkwai.com/bs2/upload-ylab-stunt/ai_portal/1732888177/cOLNrShrSO/static_mask.png" +// #[serde(skip_serializing_if = "Option::is_none")] +// pub static_mask_url: Option, +// /// URL of the image to be used for the end of the video +// #[serde(skip_serializing_if = "Option::is_none")] +// pub tail_image_url: Option, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct V1TextToVideoRequest { +// /// Advanced Camera control parameters +// #[serde(skip_serializing_if = "Option::is_none")] +// pub advanced_camera_control: Option>, +// /// The aspect ratio of the generated video frame +// #[serde(skip_serializing_if = "Option::is_none")] +// pub aspect_ratio: Option, +// /// Camera control parameters +// #[serde(skip_serializing_if = "Option::is_none")] +// pub camera_control: Option, +// /// The CFG (Classifier Free Guidance) scale is a measure of how close you want +// /// the model to stick to your prompt. +// #[serde(skip_serializing_if = "Option::is_none")] +// pub cfg_scale: Option, +// /// The duration of the generated video in seconds +// #[serde(skip_serializing_if = "Option::is_none")] +// pub duration: Option, +// #[serde(skip_serializing_if = "Option::is_none")] +// pub negative_prompt: Option, +// pub prompt: String, +// } +// +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ValidationError { + pub loc: Vec, + pub msg: String, + #[serde(rename = "type")] + pub ty: String, +} +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct VideoEffectsOutput { +// /// The generated video +// /// {"content_type":"video/mp4","file_name":"output.mp4","url":"https://storage.googleapis.com/falserverless/kling/kling_ex.mp4.mp4"} +// pub video: File, +// } +// +// #[derive(Debug, Serialize, Deserialize, Default)] +// pub struct VideoEffectsRequest { +// /// The duration of the generated video in seconds +// #[serde(skip_serializing_if = "Option::is_none")] +// pub duration: Option, +// /// The effect scene to use for the video generation +// /// "hug" +// pub effect_scene: String, +// /// URL of the image to be used for the squish and expansion video +// /// "https://h2.inkwai.com/bs2/upload-ylab-stunt/se/ai_portal_queue_mmu_image_upscale_aiweb/3214b798-e1b4-4b00-b7af-72b5b0417420_raw_image_0.jpg" +// pub image_url: String, +// /// URL of images to be used for hug, kiss or heart_gesture video. +// /// ["https://storage.googleapis.com/falserverless/juggernaut_examples/VHXMavzPyI27zi6JseyL4.png","https://storage.googleapis.com/falserverless/juggernaut_examples/QEW5VrzccxGva7mPfEXjf.png"] +// #[serde(skip_serializing_if = "Option::is_none")] +// pub input_image_urls: Option>>, +// } + +/// Kling 1.0 +/// +/// Category: text-to-video +/// Machine Type: A100 +/// +/// +/// Kling 1.6 (pro) Image to Video API. +pub fn image_to_video( + params: ProImageToVideoRequest, +) -> FalRequest { + FalRequest::new("fal-ai/kling-video/v2.1/pro/image-to-video", params) +} diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/pro/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/pro/mod.rs new file mode 100644 index 0000000000..ceee77f45a --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/kling_video/v2_1/pro/mod.rs @@ -0,0 +1,14 @@ +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_kling-video" + ))) +)] +pub mod image_to_video; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/mod.rs index ac9e8f2874..37131a3b96 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/mod.rs @@ -138,6 +138,20 @@ pub mod birefnet; ))) )] pub mod bria; +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_bytedance" + ))) +)] +pub mod bytedance; #[cfg(any( feature = "endpoints", feature = "endpoints_fal-ai", diff --git a/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx b/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx index 773cbaf7f7..03bc642558 100644 --- a/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx +++ b/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx @@ -46,6 +46,11 @@ export function LightboxModal({ onAddToSceneClicked, mediaClass, }: LightboxModalProps) { + // NB(bt,2025-06-14): We add ?cors=1 to the image url to prevent caching "sec-fetch-mode: no-cors" from + // the tag request from being cached. If we then drag it into the canvas after it's been cached, + // it won't be able to load in cors mode and will show blank in the canvas and 3D engine. This is a really + // stupid hack around this behavior. + const imageTagImageUrl = imageUrl ? imageUrl + "?cors=1" : ""; return createPortal(
@@ -102,7 +107,7 @@ export function LightboxModal({ ) : (