From ce2e545bae86669ce369478bf1fa51efc8227d53 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 20 May 2026 14:05:44 -0400 Subject: [PATCH] storyteller web initialization cleanup (#1543) --- .../service/web/storyteller_web/src/main.rs | 541 +----------------- .../src/startup/build_dependencies.rs | 339 +++++++++++ .../web/storyteller_web/src/startup/mod.rs | 8 +- .../storyteller_web/src/startup/setup_bans.rs | 68 +++ .../src/startup/setup_disabled_endpoints.rs | 25 + .../{build_pager.rs => setup_pager.rs} | 0 .../src/startup/setup_static_feature_flags.rs | 40 ++ .../src/startup/setup_stripe_artcraft.rs | 14 + .../src/startup/setup_stripe_fakeyou.rs | 35 ++ 9 files changed, 554 insertions(+), 516 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/startup/build_dependencies.rs create mode 100644 crates/service/web/storyteller_web/src/startup/setup_bans.rs create mode 100644 crates/service/web/storyteller_web/src/startup/setup_disabled_endpoints.rs rename crates/service/web/storyteller_web/src/startup/{build_pager.rs => setup_pager.rs} (100%) create mode 100644 crates/service/web/storyteller_web/src/startup/setup_static_feature_flags.rs create mode 100644 crates/service/web/storyteller_web/src/startup/setup_stripe_artcraft.rs create mode 100644 crates/service/web/storyteller_web/src/startup/setup_stripe_fakeyou.rs diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 8142d3e518..931df5f3d2 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -18,78 +18,42 @@ #[macro_use] extern crate serde_derive; use std::sync::Arc; -use std::time::Duration; use actix::Actor; use actix_artcraft::sessions::user_sessions::http_user_session_manager::HttpUserSessionManager; use actix_cors_configs::cors::build_cors_config; use actix_cors_configs::shared_array_buffer_cors::shared_array_buffer_cors; use actix_helpers::middleware::banned_cidr_filter::banned_cidr_filter::BannedCidrFilter; -use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; -use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; use actix_helpers::middleware::banned_ip_filter::banned_ip_filter::BannedIpFilter; -use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; -use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filter::DisabledEndpointFilter; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; use actix_multipart::form::MultipartFormConfig; use actix_web::middleware::{DefaultHeaders, Logger}; use actix_web::{middleware, web, App, HttpServer}; -use anyhow::anyhow; -use billing_artcraft_component::utils::artcraft_stripe_config::ArtcraftStripeConfig; -use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use chrono::Utc; -use cloud_storage::bucket_client::BucketClient; use component_traits::traits::internal_user_lookup::InternalUserLookup; -use elasticsearch::http::transport::Transport; -use elasticsearch::Elasticsearch; use errors::AnyhowResult; -use fal_client::creds::fal_api_key::FalApiKey; use log::{info, warn}; -use memory_caching::arc_ttl_sieve::ArcTtlSieve; -use memory_caching::single_item_ttl_cache::SingleItemTtlCache; -use mysql_queries::mediators::badge_granter::BadgeGranter; -use mysql_queries::mediators::firehose_publisher::FirehosePublisher; -use redis::Client; -use redis_caching::redis_ttl_cache::RedisTtlCache; use reusable_types::server_environment::ServerEnvironment; use shared_env_var_config::logging::DEFAULT_RUST_LOG; -use shared_env_var_config::redis::env_get_redis_0_connection_string_or_default; use tokio::runtime::Runtime; -use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl; use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; -use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limiters; -use crate::configs::connect_to_database::connect_to_database; -use crate::configs::static_api_tokens::StaticApiTokenSet; use crate::http_server::middleware::error_alerting_middleware::error_alerting_middleware::ErrorAlertingMiddleware; use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; use crate::http_server::routes::add_routes::add_routes; -use crate::http_server::session::session_checker::SessionChecker; use crate::http_server::web_utils::handle_multipart_error::handle_multipart_error; -use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator; -use crate::startup::build_pager::build_pager; -use crate::state::certs::google_sign_in_cert::GoogleSignInCert; -use crate::state::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; -use crate::state::server_state::{BeebleData, DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, FalData, GmiCloudData, InMemoryCaches, OpenAiData, ResendData, Seedance2ProData, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, WorldLabsData}; -use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use crate::startup::build_dependencies::setup_dependencies; +use crate::startup::setup_disabled_endpoints::read_disabled_endpoints; +use crate::state::server_state::ServerState; use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; -use crate::util::encrypted_sort_id::SortKeyCrypto; -use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; -use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; -use actix_artcraft::sessions::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; -use opaque_cursors::v2::opaque_cursor_encoder_v2::OpaqueCursorEncoderV2; pub mod billing; pub mod configs; @@ -100,28 +64,9 @@ pub mod state; pub mod threads; pub mod util; -const DEFAULT_BIND_ADDRESS : &str = "0.0.0.0:12345"; - -// Buckets (shared config) -const ENV_ACCESS_KEY : &str = "ACCESS_KEY"; -const ENV_SECRET_KEY : &str = "SECRET_KEY"; -const ENV_REGION_NAME : &str = "REGION_NAME"; - -// Buckets (private data) -const ENV_PRIVATE_BUCKET_NAME : &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; -// Buckets (public data) -const ENV_PUBLIC_BUCKET_NAME : &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; - -const ENV_GC_ENABLED_PUBLIC_BUCKET_NAME : &str = "GC_ENABLED_PUBLIC_BUCKET_NAME"; - -// Various bucket roots -const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &str = "AUDIO_UPLOADS_BUCKET_ROOT"; - // Report cloudflare trace ID header (CF-Ray) in logs -// "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; -const LOG_FORMAT : &str = +const LOG_FORMAT: &str = "[%{HOSTNAME}e] IP=[%{X-Forwarded-For}i] %{CF-Ray}i \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; -// "[%{HOSTNAME}e] IP=[%{X-Forwarded-For}i] \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; #[actix_web::main] async fn main() -> AnyhowResult<()> { @@ -140,8 +85,6 @@ async fn main() -> AnyhowResult<()> { ignore_legacy_dot_env_file: true, })?; - - info!("Obtaining hostname..."); let server_hostname = hostname::get() @@ -151,173 +94,14 @@ async fn main() -> AnyhowResult<()> { info!("Hostname: {}", &server_hostname); - info!("Connecting to database..."); + // ==================== Setup all dependencies ==================== // - let pool = connect_to_database().await?; + let setup = setup_dependencies(&server_hostname).await?; + let server_state = setup.server_state; + let health_check_interval = setup.health_check_interval; + let pager_worker = setup.pager_worker; - info!("Connected to database."); - - let firehose_publisher = FirehosePublisher { - mysql_pool: pool.clone(), // NB: Pool is clone/sync/send-safe - }; - - let badge_granter = BadgeGranter { - mysql_pool: pool.clone(), // NB: Pool is clone/sync/send-safe - firehose_publisher: firehose_publisher.clone(), // NB: Also safe - }; - - info!("Connected to redis..."); - - let redis_manager = Client::open( - env_get_redis_0_connection_string_or_default())?; - - let redis_pool = r2d2::Pool::builder() - .build(redis_manager)?; - - let redis_ttl_cache = RedisTtlCache::new_with_ttl( - redis_pool.clone(), - easyenv::get_env_num("REDIS_CACHE_TTL_SECONDS", 60)?, - ); - - info!("Connecting to elasticsearch..."); - - let elasticsearch = get_elasticsearch_client()?; - - info!("Reading env vars and setting up utils..."); - - let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); - let num_workers = easyenv::get_env_num("NUM_WORKERS", 8)?; - let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); - let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); - let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); - let cookie_http_only = easyenv::get_env_bool_or_default("COOKIE_HTTP_ONLY", true); - let website_homepage_redirect = - easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); - - let session_cookie_manager = HttpUserSessionManager::new(&cookie_domain, &hmac_secret)?; - let avt_cookie_manager = AvtCookieManager::new(&cookie_domain, &hmac_secret)?; - - let session_checker = SessionChecker::new_with_cache( - &session_cookie_manager, - redis_ttl_cache.clone(), - ); - - let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; - let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; - let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; - - // Private and Public Buckets - let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; - let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; - let gc_enabled_public_bucket_name = easyenv::get_env_string_required(ENV_GC_ENABLED_PUBLIC_BUCKET_NAME)?; - - // Bucket roots - let audio_uploads_bucket_root= easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?; - - let s3_compatible_endpoint_url = easyenv::get_env_string_or_default("S3_COMPATIBLE_ENDPOINT_URL", "https://storage.googleapis.com"); - let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS", - Duration::from_secs(60 * 5)); - - let private_bucket_client = BucketClient::create( - &access_key, - &secret_key, - ®ion_name, - &private_bucket_name, - &s3_compatible_endpoint_url, - None, - Some(bucket_timeout), - )?; - - let public_bucket_client = BucketClient::create( - &access_key, - &secret_key, - ®ion_name, - &public_bucket_name, - &s3_compatible_endpoint_url, - None, - Some(bucket_timeout), - )?; - - let auto_gc_bucket_client = BucketClient::create( - &access_key, - &secret_key, - ®ion_name, - &gc_enabled_public_bucket_name, - &s3_compatible_endpoint_url, - None, - Some(bucket_timeout), - )?; - - // In-Memory Cache - let voice_list_cache_ttl = easyenv::get_env_duration_seconds_or_default("VOICE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)); - let category_list_cache_ttl = easyenv::get_env_duration_seconds_or_default("CATEGORY_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)); - let voice_list_cache = SingleItemTtlCache::create_with_duration(voice_list_cache_ttl); - - let database_tts_category_list_cache = SingleItemTtlCache::create_with_duration( - easyenv::get_env_duration_seconds_or_default("DATABASE_TTS_CATEGORY_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)) - ); - - let w2l_template_cache = SingleItemTtlCache::create_with_duration( - easyenv::get_env_duration_seconds_or_default("W2L_TEMPLATE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(300)) // 5 minutes - ); - - let tts_queue_length_cache = SingleItemTtlCache::create_with_duration( - easyenv::get_env_duration_seconds_or_default("TTS_QUEUE_LENGTH_CACHE_TTL_SECONDS", Duration::from_secs(30)) - ); - - let tts_model_category_assignments_cache = SingleItemTtlCache::create_with_duration( - easyenv::get_env_duration_seconds_or_default("TTS_MODEL_CATEGORY_ASSIGNMENTS_CACHE_TTL_SECONDS", Duration::from_secs(60)) - ); - - let leaderboard_cache = SingleItemTtlCache::create_with_duration( - easyenv::get_env_duration_seconds_or_default("LEADERBOARD_CACHE_TTL_SECONDS", Duration::from_secs(60)) - ); - - let inference_queue_length_cache = SingleItemTtlCache::create_with_duration( - easyenv::get_env_duration_seconds_or_default("INFERENCE_QUEUE_LENGTH_CACHE_TTL_SECONDS", Duration::from_secs(30)) - ); - - // NB: This secret really isn't too important. - // We can even rotate it without too much impact to users. - let sort_key_crypto_secret = - easyenv::get_env_string_or_default("SORT_KEY_SECRET", "webscale"); - - let sort_key_crypto = SortKeyCrypto::new(&sort_key_crypto_secret); - let opaque_cursor_encoder = OpaqueCursorEncoderV2::new(&sort_key_crypto_secret); - - let health_check_interval = easyenv::get_env_duration_seconds_or_default( - "HEALTH_CHECK_INTERVAL_SECS", Duration::from_secs(3)); - - let static_api_token_set = read_static_api_tokens(); - - let ip_ban_list = load_static_container_ip_bans(); - let ip_ban_list2 = ip_ban_list.clone(); - - let cidr_ban_set = load_cidr_bans(); - - let user_token_troll_bans = load_troll_user_token_bans(); - let ip_address_troll_bans = load_ip_address_troll_bans(); - - let model_token_info_cache = ModelTokenToInfoCache::new(); - let model_token_info_cache2 = model_token_info_cache.clone(); - - // Background jobs. - - let health_check_status = HealthCheckStatus::new(); - let health_check_status2 = health_check_status.clone(); - let mysql_pool3 = pool.clone(); - let mysql_pool4 = pool.clone(); - let mysql_pool5 = pool.clone(); - - let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?) - .ok_or(anyhow!("invalid server environment"))?; - - let server_environment_typed = match server_environment { - ServerEnvironment::Production => server_environment::ServerEnvironment::Production, - ServerEnvironment::Development => server_environment::ServerEnvironment::Development, - }; - - let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed, &server_hostname); + // ==================== Background threads ==================== // let tokio_runtime = Runtime::new()?; @@ -329,310 +113,43 @@ async fn main() -> AnyhowResult<()> { info!("Spawning DB health checker thread."); - let health_checker_pager = pager.clone(); + let health_check_status_clone = server_state.health_check_status.clone(); + let mysql_pool_clone = server_state.mysql_pool.clone(); + let pager_clone = server_state.pager.clone(); tokio_runtime.spawn(async move { db_health_checker_thread( - health_check_status2, - mysql_pool3, + health_check_status_clone, + mysql_pool_clone, health_check_interval, - health_checker_pager, + pager_clone, ).await; }); info!("Spawning IP ban polling thread."); + let ip_ban_list_clone = server_state.ip_ban_list.clone(); + let mysql_pool_clone = server_state.mysql_pool.clone(); + tokio_runtime.spawn(async { - poll_ip_bans(ip_ban_list2, mysql_pool4).await; + poll_ip_bans(ip_ban_list_clone, mysql_pool_clone).await; }); info!("Spawning token info cache polling thread."); + let model_token_info_cache_clone = server_state.caches.durable.model_token_info.clone(); + let mysql_pool_clone = server_state.mysql_pool.clone(); + tokio_runtime.spawn(async { - poll_model_token_info_thread(model_token_info_cache2, mysql_pool5).await; + poll_model_token_info_thread(model_token_info_cache_clone, mysql_pool_clone).await; }); - let stripe_configs = StripeConfig { - checkout: StripeCheckoutConfigs { - success_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_CHECKOUT_SUCCESS_URL_PATH")?), - cancel_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_CHECKOUT_CANCEL_URL_PATH")?), - }, - portal: StripeCustomerPortalConfigs { - return_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_PORTAL_RETURN_URL_PATH")?), - default_portal_config_id: easyenv::get_env_string_required("STRIPE_PORTAL_DEFAULT_CONFIG_ID")?, - }, - secrets: StripeSecrets { - publishable_key: easyenv::get_env_string_optional("STRIPE_PUBLISHABLE_KEY"), - secret_key: easyenv::get_env_string_required("STRIPE_SECRET_KEY")?, - secret_webhook_signing_key: easyenv::get_env_string_required("STRIPE_SECRET_WEBHOOK_SIGNING_KEY")?, - }, - }; + // ==================== Serve ==================== // - let service_feature_flags = StaticFeatureFlags { - // Permanent (control plane / safety) flags : messaging - maybe_status_alert_category: easyenv::get_env_string_optional("FF_STATUS_ALERT_CATEGORY"), - maybe_status_alert_custom_message: easyenv::get_env_string_optional("FF_STATUS_ALERT_CUSTOM_MESSAGE"), - - // Permanent (control plane / safety) flags : disabling features - global_429_pushback_filter_enabled: easyenv::get_env_bool_or_default("FF_GLOBAL_429_PUSHBACK_FILTER_ENABLED", false), - disable_unified_queue_stats_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_QUEUE_STATS_ENDPOINT", false), - disable_inference_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_INFERENCE_QUEUE_LENGTH_ENDPOINT", false), - disable_tts_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_QUEUE_LENGTH_ENDPOINT", false), - disable_tts_model_list_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_MODEL_LIST_ENDPOINT", false), - disable_voice_conversion_model_list_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_VOICE_CONVERSION_MODEL_LIST_ENDPOINT", false), - - // Refresh rates - frontend_unified_queue_stats_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_QUEUE_STATS_REFRESH_INTERVAL_MILLIS", 15_000)?, - frontend_pending_inference_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_INFERENCE_REFRESH_INTERVAL_MILLIS", 15_000)?, - frontend_pending_tts_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_TTS_REFRESH_INTERVAL_MILLIS", 15_000)?, - - // Bans - troll_ban_user_percent: easyenv::get_env_num("FF_TROLL_BANNED_USER_PERCENT", 0)?, - - // Temporary flags - switch_tts_to_model_weights: easyenv::get_env_bool_or_default("FF_SWITCH_TTS_TO_MODEL_WEIGHTS", false), - force_session_studio_flags: easyenv::get_env_bool_or_default("FF_FORCE_SESSION_STUDIO_FLAG", false), - force_session_video_style_transfer_flags: easyenv::get_env_bool_or_default("FF_FORCE_SESSION_VST_FLAG", false), - - // Disable voice features - disable_tts: easyenv::get_env_bool_or_default("DISABLE_TTS", false), - disable_voice_conversion: easyenv::get_env_bool_or_default("DISABLE_VOICE_CONVERSION", false), - - // Paging - paging: paging_flags, - }; - - let third_party_url_redirector = ThirdPartyUrlRedirector::new(server_environment); - - let stripe_client = { - let api_secret = stripe_configs.secrets.secret_key.clone(); - stripe::Client::new(api_secret) - }; - - // NB: Docker creates this file within container builds. - let build_sha = std::fs::read_to_string("/GIT_SHA") - .unwrap_or(String::from("unknown")) - .trim() - .to_string(); - - let fal_api_key = FalApiKey::new(easyenv::get_env_string_required("FAL_API_KEY")?); - let fal_webhook_url = easyenv::get_env_string_required("FAL_WEBHOOK_URL")?; - - let openai_api_key= easyenv::get_env_string_required("OPENAI_API_KEY")?; - - let resend_api_key = easyenv::get_env_string_required("RESEND_API_KEY")?; - - let gmicloud_api_key = easyenv::get_env_string_required("GMICLOUD_API_KEY")?; - - let worldlabs_api_key = easyenv::get_env_string_required("WORLDLABS_API_KEY")?; - - let startup_time = Utc::now(); - - let server_state = ServerState { - env_config: EnvConfig { - num_workers, - bind_address, - cookie_domain, - cookie_secure, - cookie_http_only, - website_homepage_redirect, - }, - server_info: ServerInfo { - build_sha, - }, - stripe: StripeSettings { - config: stripe_configs, - client: stripe_client, - }, - stripe_artcraft: { - ArtcraftStripeConfig { - secret_key: easyenv::get_env_string_required("STRIPE_ARTCRAFT_SECRET_KEY")?, - secret_webhook_signing_key: easyenv::get_env_string_required("STRIPE_ARTCRAFT_SECRET_WEBHOOK_KEY")?, - checkout_success_url: easyenv::get_env_string_required("STRIPE_ARTCRAFT_CHECKOUT_SUCCESS_URL")?, - checkout_cancel_url: easyenv::get_env_string_required("STRIPE_ARTCRAFT_CHECKOUT_CANCEL_URL")?, - portal_return_url: easyenv::get_env_string_required("STRIPE_ARTCRAFT_PORTAL_RETURN_URL")?, - }.to_config_with_client() - }, - hostname: server_hostname, - startup_time, - server_environment_old: server_environment, - server_environment: server_environment_typed, - flags: service_feature_flags, - third_party_url_redirector, - health_check_status, - mysql_pool: pool, - elasticsearch, - redis_pool, - redis_ttl_cache, - redis_rate_limiters: configure_redis_rate_limiters()?, - firehose_publisher, - badge_granter, - avt_cookie_manager, - session_cookie_manager, - session_checker, - private_bucket_client, - public_bucket_client, - auto_gc_bucket_client, - audio_uploads_bucket_root, - sort_key_crypto, - opaque_cursors: opaque_cursor_encoder, - static_api_token_set, - beeble: BeebleData { - api_key: beeble_client::creds::beeble_api_key::BeebleApiKey::new( - easyenv::get_env_string_required("BEEBLE_API_KEY")?, - ), - webhook_url: easyenv::get_env_string_required("BEEBLE_WEBHOOK_URL")?, - }, - fal: FalData { - api_key: fal_api_key, - webhook_url: fal_webhook_url, - }, - gmicloud: GmiCloudData { - api_key: gmicloud_client::creds::gmicloud_api_key::GmiCloudApiKey::new(gmicloud_api_key), - }, - seedance2pro: Seedance2ProData { - cookies: easyenv::get_env_string_required("SEEDANCE2PRO_COOKIES")?, - cookies_byteplus: easyenv::get_env_string_required("SEEDANCE2PRO_WHITELIST_COOKIES")?, - }, - openai: OpenAiData { - api_key: openai_api_key, - }, - resend: ResendData { - api_key: resend_api_key, - }, - worldlabs: WorldLabsData { - api_key: worldlabs_api_key, - }, - pager, - caches: InMemoryCaches { - durable: DurableInMemoryCaches { - model_token_info: model_token_info_cache, - }, - ephemeral: EphemeralInMemoryCaches { - tts_model_list: voice_list_cache, - voice_conversion_model_list: SingleItemTtlCache::create_with_duration( - easyenv::get_env_duration_seconds_or_default( - "VOICE_CONVERSION_MODEL_LIST_CACHE_TTL_SECONDS", - Duration::from_secs(60))), - w2l_template_list: w2l_template_cache, - database_tts_category_list: database_tts_category_list_cache, - tts_queue_length: tts_queue_length_cache, - tts_model_category_assignments: tts_model_category_assignments_cache, - leaderboard: leaderboard_cache, - inference_queue_length: inference_queue_length_cache, - queue_stats: SingleItemTtlCache::create_with_duration( - easyenv::get_env_duration_seconds_or_default( - "QUEUE_STATS_CACHE_TTL_SECONDS", - Duration::from_secs(60))), - featured_media_files_sieve: ArcTtlSieve::with_capacity_and_ttl_duration( - easyenv::get_env_num("FEATURED_MEDIA_FILES_CACHE_SIZE", 25)?, - easyenv::get_env_duration_seconds_or_default("FEATURED_MEDIA_FILES_TTL_SECONDS", Duration::from_secs(60)), - )?, - } - }, - ip_ban_list, - cidr_ban_set, - troll_bans: TrollBans { - user_tokens: user_token_troll_bans, - ip_addresses: ip_address_troll_bans, - }, - temp_dir_creator: ScopedTempDirCreator::auto_setup(), - google_sign_in_cert: GoogleSignInCert::new(), - }; - - serve(server_state) - .await?; + serve(server_state).await?; Ok(()) } -fn read_static_api_tokens() -> StaticApiTokenSet { - let filename = easyenv::get_env_string_or_default( - "STATIC_API_TOKENS_CONFIG_FILE", - "./configs/static_api_tokens.toml"); - - StaticApiTokenSet::from_file(&filename) -} - -fn read_disabled_endpoints() -> DisabledEndpoints { - let exact_filename = easyenv::get_env_string_or_default( - "DISABLED_ENDPOINTS_FILE_EXACT_MATCH", - "./includes/container_includes/disabled_endpoints/endpoint_exact_matches.txt"); - - let exact = ExactMatchDisabledEndpoints::load_from_file(exact_filename) - .unwrap_or(ExactMatchDisabledEndpoints::new()); // NB: Fail open - - let prefix_filename = easyenv::get_env_string_or_default( - "DISABLED_ENDPOINTS_FILE_PREFIX_MATCH", - "./includes/container_includes/disabled_endpoints/endpoint_prefixes.txt"); - - let prefix = PrefixDisabledEndpoints::load_from_file(prefix_filename) - .unwrap_or(PrefixDisabledEndpoints::new()); // NB: Fail open - - info!("Disabled endpoints by exact match: {}", exact.len()); - info!("Disabled endpoints by prefix: {}", prefix.len()); - - DisabledEndpoints::new(exact, prefix) -} - -fn load_static_container_ip_bans() -> IpBanList { - let ip_ban_directory = easyenv::get_env_string_or_default( - "IP_BAN_DIRECTORY", - "./includes/container_includes/banned_ip_addresses" - ); - - let ip_ban_list = load_ip_ban_list_from_directory(ip_ban_directory) - .unwrap_or(IpBanList::new()); - - info!("Static IP bans loaded: {}", ip_ban_list.total_ip_address_count().unwrap_or(0)); - ip_ban_list -} - -fn load_cidr_bans() -> BannedCidrSet { - let cidr_ban_file = easyenv::get_env_string_or_default( - "CIDR_BAN_FILE", - "./includes/container_includes/banned_cidrs/banned_cidrs.txt" - ); - - let cidr_bans = load_cidr_ban_set_from_file(cidr_ban_file) - .unwrap_or(BannedCidrSet::new()); - - info!("CIDR bans loaded : {} CIDRs, {} addresses total", - cidr_bans.total_cidr_count().unwrap_or(0), - cidr_bans.total_ip_address_count().unwrap_or(0)); - - cidr_bans -} - -// NB: Some users abuse our service. -// Instead of outright banning them, we can change the function of the service. -fn load_troll_user_token_bans() -> TrollUserBanList { - let user_token_troll_ban_directory = easyenv::get_env_string_or_default( - "USER_TOKEN_TROLL_BAN_DIRECTORY", - "./includes/container_includes/troll_bans/user_token_troll_bans" - ); - - let troll_ban_list = load_user_token_ban_list_from_directory(user_token_troll_ban_directory) - .unwrap_or(TrollUserBanList::new()); - - info!("Static user token troll bans loaded: {}", troll_ban_list.total_user_token_count().unwrap_or(0)); - troll_ban_list -} - -// NB: Some users abuse our service. -// Instead of outright banning them, we can change the function of the service. -fn load_ip_address_troll_bans() -> IpBanList { - let ip_ban_directory = easyenv::get_env_string_or_default( - "IP_TROLL_BAN_DIRECTORY", - "./includes/container_includes/troll_bans/ip_address_troll_bans" - ); - - let ip_ban_list = load_ip_ban_list_from_directory(ip_ban_directory) - .unwrap_or(IpBanList::new()); - - info!("Static IP troll bans loaded: {}", ip_ban_list.total_ip_address_count().unwrap_or(0)); - ip_ban_list -} - pub async fn serve(server_state: ServerState) -> AnyhowResult<()> { let bind_address = server_state.env_config.bind_address.clone(); @@ -739,9 +256,3 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> Ok(()) } - -fn get_elasticsearch_client() -> AnyhowResult { - let transport = Transport::single_node(&easyenv::get_env_string_required("ELASTICSEARCH_URL")?)?; - let client = Elasticsearch::new(transport); - Ok(client) -} diff --git a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs new file mode 100644 index 0000000000..d3b9191402 --- /dev/null +++ b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs @@ -0,0 +1,339 @@ +use std::sync::Arc; +use std::time::Duration; + +use actix_artcraft::sessions::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; +use actix_artcraft::sessions::user_sessions::http_user_session_manager::HttpUserSessionManager; +use anyhow::anyhow; +use chrono::Utc; +use cloud_storage::bucket_client::BucketClient; +use elasticsearch::http::transport::Transport; +use elasticsearch::Elasticsearch; +use errors::AnyhowResult; +use fal_client::creds::fal_api_key::FalApiKey; +use log::info; +use memory_caching::arc_ttl_sieve::ArcTtlSieve; +use memory_caching::single_item_ttl_cache::SingleItemTtlCache; +use mysql_queries::mediators::badge_granter::BadgeGranter; +use mysql_queries::mediators::firehose_publisher::FirehosePublisher; +use opaque_cursors::v2::opaque_cursor_encoder_v2::OpaqueCursorEncoderV2; +use pager::worker::pager_worker::PagerWorker; +use redis::Client; +use redis_caching::redis_ttl_cache::RedisTtlCache; +use reusable_types::server_environment::ServerEnvironment; +use shared_env_var_config::redis::env_get_redis_0_connection_string_or_default; +use sqlx::MySqlPool; +use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; + +use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limiters; +use crate::configs::connect_to_database::connect_to_database; +use crate::configs::static_api_tokens::StaticApiTokenSet; +use crate::http_server::session::session_checker::SessionChecker; +use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator; +use crate::startup::setup_pager::build_pager; +use crate::startup::setup_bans::{ + load_cidr_bans, load_ip_address_troll_bans, + load_static_container_ip_bans, load_troll_user_token_bans, +}; +use crate::startup::setup_static_feature_flags::setup_static_feature_flags; +use crate::startup::setup_stripe_artcraft::setup_stripe_artcraft; +use crate::startup::setup_stripe_fakeyou::setup_stripe_fakeyou; +use crate::state::certs::google_sign_in_cert::GoogleSignInCert; +use crate::state::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; +use crate::state::server_state::{ + BeebleData, DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, FalData, + GmiCloudData, InMemoryCaches, OpenAiData, ResendData, Seedance2ProData, ServerInfo, + ServerState, TrollBans, WorldLabsData, +}; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use crate::util::encrypted_sort_id::SortKeyCrypto; + +// Bucket config +const ENV_ACCESS_KEY: &str = "ACCESS_KEY"; +const ENV_SECRET_KEY: &str = "SECRET_KEY"; +const ENV_REGION_NAME: &str = "REGION_NAME"; +const ENV_PRIVATE_BUCKET_NAME: &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +const ENV_PUBLIC_BUCKET_NAME: &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; +const ENV_GC_ENABLED_PUBLIC_BUCKET_NAME: &str = "GC_ENABLED_PUBLIC_BUCKET_NAME"; +const ENV_AUDIO_UPLOADS_BUCKET_ROOT: &str = "AUDIO_UPLOADS_BUCKET_ROOT"; + +/// Everything produced by server startup that the main function needs. +pub struct SetupResult { + pub server_state: ServerState, + pub pager_worker: PagerWorker, + pub health_check_interval: Duration, +} + +pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult { + info!("Connecting to database..."); + let pool = connect_to_database().await?; + + info!("Connected to database."); + + let firehose_publisher = FirehosePublisher { + mysql_pool: pool.clone(), + }; + + let badge_granter = BadgeGranter { + mysql_pool: pool.clone(), + firehose_publisher: firehose_publisher.clone(), + }; + + info!("Connecting to redis..."); + + let redis_manager = Client::open(env_get_redis_0_connection_string_or_default())?; + let redis_pool = r2d2::Pool::builder().build(redis_manager)?; + + let redis_ttl_cache = RedisTtlCache::new_with_ttl( + redis_pool.clone(), + easyenv::get_env_num("REDIS_CACHE_TTL_SECONDS", 60)?, + ); + + info!("Connecting to elasticsearch..."); + let elasticsearch = get_elasticsearch_client()?; + + info!("Reading env vars and setting up utils..."); + + let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", "0.0.0.0:12345"); + let num_workers = easyenv::get_env_num("NUM_WORKERS", 8)?; + let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); + let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); + let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); + let cookie_http_only = easyenv::get_env_bool_or_default("COOKIE_HTTP_ONLY", true); + let website_homepage_redirect = + easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); + + let session_cookie_manager = HttpUserSessionManager::new(&cookie_domain, &hmac_secret)?; + let avt_cookie_manager = AvtCookieManager::new(&cookie_domain, &hmac_secret)?; + + let session_checker = SessionChecker::new_with_cache( + &session_cookie_manager, + redis_ttl_cache.clone(), + ); + + let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; + let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; + let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; + + let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; + let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + let gc_enabled_public_bucket_name = easyenv::get_env_string_required(ENV_GC_ENABLED_PUBLIC_BUCKET_NAME)?; + + let audio_uploads_bucket_root = easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?; + + let s3_compatible_endpoint_url = easyenv::get_env_string_or_default("S3_COMPATIBLE_ENDPOINT_URL", "https://storage.googleapis.com"); + let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5)); + + let private_bucket_client = BucketClient::create( + &access_key, &secret_key, ®ion_name, &private_bucket_name, + &s3_compatible_endpoint_url, None, Some(bucket_timeout), + )?; + + let public_bucket_client = BucketClient::create( + &access_key, &secret_key, ®ion_name, &public_bucket_name, + &s3_compatible_endpoint_url, None, Some(bucket_timeout), + )?; + + let auto_gc_bucket_client = BucketClient::create( + &access_key, &secret_key, ®ion_name, &gc_enabled_public_bucket_name, + &s3_compatible_endpoint_url, None, Some(bucket_timeout), + )?; + + // In-Memory Caches + let voice_list_cache_ttl = easyenv::get_env_duration_seconds_or_default("VOICE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)); + let voice_list_cache = SingleItemTtlCache::create_with_duration(voice_list_cache_ttl); + + let database_tts_category_list_cache = SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default("DATABASE_TTS_CATEGORY_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)) + ); + + let w2l_template_cache = SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default("W2L_TEMPLATE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(300)) + ); + + let tts_queue_length_cache = SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default("TTS_QUEUE_LENGTH_CACHE_TTL_SECONDS", Duration::from_secs(30)) + ); + + let tts_model_category_assignments_cache = SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default("TTS_MODEL_CATEGORY_ASSIGNMENTS_CACHE_TTL_SECONDS", Duration::from_secs(60)) + ); + + let leaderboard_cache = SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default("LEADERBOARD_CACHE_TTL_SECONDS", Duration::from_secs(60)) + ); + + let inference_queue_length_cache = SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default("INFERENCE_QUEUE_LENGTH_CACHE_TTL_SECONDS", Duration::from_secs(30)) + ); + + let sort_key_crypto_secret = easyenv::get_env_string_or_default("SORT_KEY_SECRET", "webscale"); + let sort_key_crypto = SortKeyCrypto::new(&sort_key_crypto_secret); + let opaque_cursor_encoder = OpaqueCursorEncoderV2::new(&sort_key_crypto_secret); + + let health_check_interval = easyenv::get_env_duration_seconds_or_default( + "HEALTH_CHECK_INTERVAL_SECS", Duration::from_secs(3)); + + let static_api_token_set = read_static_api_tokens(); + + let ip_ban_list = load_static_container_ip_bans(); + + let cidr_ban_set = load_cidr_bans(); + + let user_token_troll_bans = load_troll_user_token_bans(); + let ip_address_troll_bans = load_ip_address_troll_bans(); + + let model_token_info_cache = ModelTokenToInfoCache::new(); + + let health_check_status = HealthCheckStatus::new(); + + let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?) + .ok_or(anyhow!("invalid server environment"))?; + + let server_environment_typed = match server_environment { + ServerEnvironment::Production => server_environment::ServerEnvironment::Production, + ServerEnvironment::Development => server_environment::ServerEnvironment::Development, + }; + + let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed, server_hostname); + + let service_feature_flags = setup_static_feature_flags(paging_flags)?; + + let third_party_url_redirector = ThirdPartyUrlRedirector::new(server_environment); + + // NB: Docker creates this file within container builds. + let build_sha = std::fs::read_to_string("/GIT_SHA") + .unwrap_or(String::from("unknown")) + .trim() + .to_string(); + + let fal_api_key = FalApiKey::new(easyenv::get_env_string_required("FAL_API_KEY")?); + let fal_webhook_url = easyenv::get_env_string_required("FAL_WEBHOOK_URL")?; + + let openai_api_key = easyenv::get_env_string_required("OPENAI_API_KEY")?; + let resend_api_key = easyenv::get_env_string_required("RESEND_API_KEY")?; + let gmicloud_api_key = easyenv::get_env_string_required("GMICLOUD_API_KEY")?; + let worldlabs_api_key = easyenv::get_env_string_required("WORLDLABS_API_KEY")?; + + let startup_time = Utc::now(); + + let server_state = ServerState { + env_config: EnvConfig { + num_workers, + bind_address, + cookie_domain, + cookie_secure, + cookie_http_only, + website_homepage_redirect, + }, + server_info: ServerInfo { + build_sha, + }, + stripe: setup_stripe_fakeyou()?, + stripe_artcraft: setup_stripe_artcraft()?, + hostname: server_hostname.to_string(), + startup_time, + server_environment_old: server_environment, + server_environment: server_environment_typed, + flags: service_feature_flags, + third_party_url_redirector, + health_check_status, + mysql_pool: pool, + elasticsearch, + redis_pool, + redis_ttl_cache, + redis_rate_limiters: configure_redis_rate_limiters()?, + firehose_publisher, + badge_granter, + avt_cookie_manager, + session_cookie_manager, + session_checker, + private_bucket_client, + public_bucket_client, + auto_gc_bucket_client, + audio_uploads_bucket_root, + sort_key_crypto, + opaque_cursors: opaque_cursor_encoder, + static_api_token_set, + beeble: BeebleData { + api_key: beeble_client::creds::beeble_api_key::BeebleApiKey::new( + easyenv::get_env_string_required("BEEBLE_API_KEY")?, + ), + webhook_url: easyenv::get_env_string_required("BEEBLE_WEBHOOK_URL")?, + }, + fal: FalData { + api_key: fal_api_key, + webhook_url: fal_webhook_url, + }, + gmicloud: GmiCloudData { + api_key: gmicloud_client::creds::gmicloud_api_key::GmiCloudApiKey::new(gmicloud_api_key), + }, + seedance2pro: Seedance2ProData { + cookies: easyenv::get_env_string_required("SEEDANCE2PRO_COOKIES")?, + cookies_byteplus: easyenv::get_env_string_required("SEEDANCE2PRO_WHITELIST_COOKIES")?, + }, + openai: OpenAiData { + api_key: openai_api_key, + }, + resend: ResendData { + api_key: resend_api_key, + }, + worldlabs: WorldLabsData { + api_key: worldlabs_api_key, + }, + pager, + caches: InMemoryCaches { + durable: DurableInMemoryCaches { + model_token_info: model_token_info_cache, + }, + ephemeral: EphemeralInMemoryCaches { + tts_model_list: voice_list_cache, + voice_conversion_model_list: SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default( + "VOICE_CONVERSION_MODEL_LIST_CACHE_TTL_SECONDS", + Duration::from_secs(60))), + w2l_template_list: w2l_template_cache, + database_tts_category_list: database_tts_category_list_cache, + tts_queue_length: tts_queue_length_cache, + tts_model_category_assignments: tts_model_category_assignments_cache, + leaderboard: leaderboard_cache, + inference_queue_length: inference_queue_length_cache, + queue_stats: SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default( + "QUEUE_STATS_CACHE_TTL_SECONDS", + Duration::from_secs(60))), + featured_media_files_sieve: ArcTtlSieve::with_capacity_and_ttl_duration( + easyenv::get_env_num("FEATURED_MEDIA_FILES_CACHE_SIZE", 25)?, + easyenv::get_env_duration_seconds_or_default("FEATURED_MEDIA_FILES_TTL_SECONDS", Duration::from_secs(60)), + )?, + } + }, + ip_ban_list, + cidr_ban_set, + troll_bans: TrollBans { + user_tokens: user_token_troll_bans, + ip_addresses: ip_address_troll_bans, + }, + temp_dir_creator: ScopedTempDirCreator::auto_setup(), + google_sign_in_cert: GoogleSignInCert::new(), + }; + + Ok(SetupResult { + server_state, + pager_worker, + health_check_interval, + }) +} + +fn read_static_api_tokens() -> StaticApiTokenSet { + let filename = easyenv::get_env_string_or_default( + "STATIC_API_TOKENS_CONFIG_FILE", + "./configs/static_api_tokens.toml"); + + StaticApiTokenSet::from_file(&filename) +} + +fn get_elasticsearch_client() -> AnyhowResult { + let transport = Transport::single_node(&easyenv::get_env_string_required("ELASTICSEARCH_URL")?)?; + let client = Elasticsearch::new(transport); + Ok(client) +} diff --git a/crates/service/web/storyteller_web/src/startup/mod.rs b/crates/service/web/storyteller_web/src/startup/mod.rs index d074c6c0ce..0c5e1eb5ec 100644 --- a/crates/service/web/storyteller_web/src/startup/mod.rs +++ b/crates/service/web/storyteller_web/src/startup/mod.rs @@ -1 +1,7 @@ -pub mod build_pager; +pub mod build_dependencies; +pub mod setup_bans; +pub mod setup_pager; +pub mod setup_disabled_endpoints; +pub mod setup_static_feature_flags; +pub mod setup_stripe_artcraft; +pub mod setup_stripe_fakeyou; diff --git a/crates/service/web/storyteller_web/src/startup/setup_bans.rs b/crates/service/web/storyteller_web/src/startup/setup_bans.rs new file mode 100644 index 0000000000..75fbdf2621 --- /dev/null +++ b/crates/service/web/storyteller_web/src/startup/setup_bans.rs @@ -0,0 +1,68 @@ +use log::info; + +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; +use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; + +use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; +use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; + +pub fn load_static_container_ip_bans() -> IpBanList { + let ip_ban_directory = easyenv::get_env_string_or_default( + "IP_BAN_DIRECTORY", + "./includes/container_includes/banned_ip_addresses" + ); + + let ip_ban_list = load_ip_ban_list_from_directory(ip_ban_directory) + .unwrap_or(IpBanList::new()); + + info!("Static IP bans loaded: {}", ip_ban_list.total_ip_address_count().unwrap_or(0)); + ip_ban_list +} + +pub fn load_cidr_bans() -> BannedCidrSet { + let cidr_ban_file = easyenv::get_env_string_or_default( + "CIDR_BAN_FILE", + "./includes/container_includes/banned_cidrs/banned_cidrs.txt" + ); + + let cidr_bans = load_cidr_ban_set_from_file(cidr_ban_file) + .unwrap_or(BannedCidrSet::new()); + + info!("CIDR bans loaded : {} CIDRs, {} addresses total", + cidr_bans.total_cidr_count().unwrap_or(0), + cidr_bans.total_ip_address_count().unwrap_or(0)); + + cidr_bans +} + +// NB: Some users abuse our service. +// Instead of outright banning them, we can change the function of the service. +pub fn load_troll_user_token_bans() -> TrollUserBanList { + let user_token_troll_ban_directory = easyenv::get_env_string_or_default( + "USER_TOKEN_TROLL_BAN_DIRECTORY", + "./includes/container_includes/troll_bans/user_token_troll_bans" + ); + + let troll_ban_list = load_user_token_ban_list_from_directory(user_token_troll_ban_directory) + .unwrap_or(TrollUserBanList::new()); + + info!("Static user token troll bans loaded: {}", troll_ban_list.total_user_token_count().unwrap_or(0)); + troll_ban_list +} + +// NB: Some users abuse our service. +// Instead of outright banning them, we can change the function of the service. +pub fn load_ip_address_troll_bans() -> IpBanList { + let ip_ban_directory = easyenv::get_env_string_or_default( + "IP_TROLL_BAN_DIRECTORY", + "./includes/container_includes/troll_bans/ip_address_troll_bans" + ); + + let ip_ban_list = load_ip_ban_list_from_directory(ip_ban_directory) + .unwrap_or(IpBanList::new()); + + info!("Static IP troll bans loaded: {}", ip_ban_list.total_ip_address_count().unwrap_or(0)); + ip_ban_list +} diff --git a/crates/service/web/storyteller_web/src/startup/setup_disabled_endpoints.rs b/crates/service/web/storyteller_web/src/startup/setup_disabled_endpoints.rs new file mode 100644 index 0000000000..c37b80600e --- /dev/null +++ b/crates/service/web/storyteller_web/src/startup/setup_disabled_endpoints.rs @@ -0,0 +1,25 @@ +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; +use log::info; + +pub fn read_disabled_endpoints() -> DisabledEndpoints { + let exact_filename = easyenv::get_env_string_or_default( + "DISABLED_ENDPOINTS_FILE_EXACT_MATCH", + "./includes/container_includes/disabled_endpoints/endpoint_exact_matches.txt"); + + let exact = ExactMatchDisabledEndpoints::load_from_file(exact_filename) + .unwrap_or(ExactMatchDisabledEndpoints::new()); // NB: Fail open + + let prefix_filename = easyenv::get_env_string_or_default( + "DISABLED_ENDPOINTS_FILE_PREFIX_MATCH", + "./includes/container_includes/disabled_endpoints/endpoint_prefixes.txt"); + + let prefix = PrefixDisabledEndpoints::load_from_file(prefix_filename) + .unwrap_or(PrefixDisabledEndpoints::new()); // NB: Fail open + + info!("Disabled endpoints by exact match: {}", exact.len()); + info!("Disabled endpoints by prefix: {}", prefix.len()); + + DisabledEndpoints::new(exact, prefix) +} diff --git a/crates/service/web/storyteller_web/src/startup/build_pager.rs b/crates/service/web/storyteller_web/src/startup/setup_pager.rs similarity index 100% rename from crates/service/web/storyteller_web/src/startup/build_pager.rs rename to crates/service/web/storyteller_web/src/startup/setup_pager.rs diff --git a/crates/service/web/storyteller_web/src/startup/setup_static_feature_flags.rs b/crates/service/web/storyteller_web/src/startup/setup_static_feature_flags.rs new file mode 100644 index 0000000000..5ad9215a17 --- /dev/null +++ b/crates/service/web/storyteller_web/src/startup/setup_static_feature_flags.rs @@ -0,0 +1,40 @@ +use errors::AnyhowResult; + +use crate::state::flags::paging_flags::PagingFlags; +use crate::state::server_state::StaticFeatureFlags; + +pub fn setup_static_feature_flags(paging_flags: PagingFlags) -> AnyhowResult { + Ok(StaticFeatureFlags { + // Permanent (control plane / safety) flags : messaging + maybe_status_alert_category: easyenv::get_env_string_optional("FF_STATUS_ALERT_CATEGORY"), + maybe_status_alert_custom_message: easyenv::get_env_string_optional("FF_STATUS_ALERT_CUSTOM_MESSAGE"), + + // Permanent (control plane / safety) flags : disabling features + global_429_pushback_filter_enabled: easyenv::get_env_bool_or_default("FF_GLOBAL_429_PUSHBACK_FILTER_ENABLED", false), + disable_unified_queue_stats_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_QUEUE_STATS_ENDPOINT", false), + disable_inference_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_INFERENCE_QUEUE_LENGTH_ENDPOINT", false), + disable_tts_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_QUEUE_LENGTH_ENDPOINT", false), + disable_tts_model_list_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_MODEL_LIST_ENDPOINT", false), + disable_voice_conversion_model_list_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_VOICE_CONVERSION_MODEL_LIST_ENDPOINT", false), + + // Refresh rates + frontend_unified_queue_stats_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_QUEUE_STATS_REFRESH_INTERVAL_MILLIS", 15_000)?, + frontend_pending_inference_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_INFERENCE_REFRESH_INTERVAL_MILLIS", 15_000)?, + frontend_pending_tts_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_TTS_REFRESH_INTERVAL_MILLIS", 15_000)?, + + // Bans + troll_ban_user_percent: easyenv::get_env_num("FF_TROLL_BANNED_USER_PERCENT", 0)?, + + // Temporary flags + switch_tts_to_model_weights: easyenv::get_env_bool_or_default("FF_SWITCH_TTS_TO_MODEL_WEIGHTS", false), + force_session_studio_flags: easyenv::get_env_bool_or_default("FF_FORCE_SESSION_STUDIO_FLAG", false), + force_session_video_style_transfer_flags: easyenv::get_env_bool_or_default("FF_FORCE_SESSION_VST_FLAG", false), + + // Disable voice features + disable_tts: easyenv::get_env_bool_or_default("DISABLE_TTS", false), + disable_voice_conversion: easyenv::get_env_bool_or_default("DISABLE_VOICE_CONVERSION", false), + + // Paging + paging: paging_flags, + }) +} diff --git a/crates/service/web/storyteller_web/src/startup/setup_stripe_artcraft.rs b/crates/service/web/storyteller_web/src/startup/setup_stripe_artcraft.rs new file mode 100644 index 0000000000..23c76d9a01 --- /dev/null +++ b/crates/service/web/storyteller_web/src/startup/setup_stripe_artcraft.rs @@ -0,0 +1,14 @@ +use billing_artcraft_component::utils::artcraft_stripe_config::{ + ArtcraftStripeConfig, ArtcraftStripeConfigWithClient, +}; +use errors::AnyhowResult; + +pub fn setup_stripe_artcraft() -> AnyhowResult { + Ok(ArtcraftStripeConfig { + secret_key: easyenv::get_env_string_required("STRIPE_ARTCRAFT_SECRET_KEY")?, + secret_webhook_signing_key: easyenv::get_env_string_required("STRIPE_ARTCRAFT_SECRET_WEBHOOK_KEY")?, + checkout_success_url: easyenv::get_env_string_required("STRIPE_ARTCRAFT_CHECKOUT_SUCCESS_URL")?, + checkout_cancel_url: easyenv::get_env_string_required("STRIPE_ARTCRAFT_CHECKOUT_CANCEL_URL")?, + portal_return_url: easyenv::get_env_string_required("STRIPE_ARTCRAFT_PORTAL_RETURN_URL")?, + }.to_config_with_client()) +} diff --git a/crates/service/web/storyteller_web/src/startup/setup_stripe_fakeyou.rs b/crates/service/web/storyteller_web/src/startup/setup_stripe_fakeyou.rs new file mode 100644 index 0000000000..694485f264 --- /dev/null +++ b/crates/service/web/storyteller_web/src/startup/setup_stripe_fakeyou.rs @@ -0,0 +1,35 @@ +use billing_component::stripe::stripe_config::{ + FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, + StripeCustomerPortalConfigs, StripeSecrets, +}; +use errors::AnyhowResult; + +use crate::state::server_state::StripeSettings; + +pub fn setup_stripe_fakeyou() -> AnyhowResult { + let stripe_configs = StripeConfig { + checkout: StripeCheckoutConfigs { + success_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_CHECKOUT_SUCCESS_URL_PATH")?), + cancel_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_CHECKOUT_CANCEL_URL_PATH")?), + }, + portal: StripeCustomerPortalConfigs { + return_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_PORTAL_RETURN_URL_PATH")?), + default_portal_config_id: easyenv::get_env_string_required("STRIPE_PORTAL_DEFAULT_CONFIG_ID")?, + }, + secrets: StripeSecrets { + publishable_key: easyenv::get_env_string_optional("STRIPE_PUBLISHABLE_KEY"), + secret_key: easyenv::get_env_string_required("STRIPE_SECRET_KEY")?, + secret_webhook_signing_key: easyenv::get_env_string_required("STRIPE_SECRET_WEBHOOK_SIGNING_KEY")?, + }, + }; + + let stripe_client = { + let api_secret = stripe_configs.secrets.secret_key.clone(); + stripe::Client::new(api_secret) + }; + + Ok(StripeSettings { + config: stripe_configs, + client: stripe_client, + }) +}