diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index 7393f389fc..5689e725cd 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -33,6 +33,7 @@ retryably scrollback softvc + spammy sql's symphonia tacotron diff --git a/Cargo.lock b/Cargo.lock index b4731a91d2..fcde783143 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,9 +20,9 @@ dependencies = [ "log", "once_cell", "parking_lot 0.11.1", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "smallvec 1.10.0", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-util 0.6.9", ] @@ -53,8 +53,8 @@ dependencies = [ "futures-core", "futures-sink", "memchr", - "pin-project-lite 0.2.9", - "tokio 1.25.0", + "pin-project-lite 0.2.13", + "tokio 1.34.0", "tokio-util 0.7.4", "tracing", ] @@ -127,7 +127,7 @@ dependencies = [ "mime", "mime_guess", "percent-encoding", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", ] [[package]] @@ -204,11 +204,11 @@ dependencies = [ "local-channel", "mime", "percent-encoding", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "rand 0.8.5", "sha1 0.10.4", "smallvec 1.10.0", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-util 0.7.4", "tracing", "zstd 0.12.3+zstd.1.5.2", @@ -302,7 +302,7 @@ checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "actix-macros 0.2.3", "futures-core", - "tokio 1.25.0", + "tokio 1.34.0", ] [[package]] @@ -336,10 +336,10 @@ dependencies = [ "actix-utils 3.0.0", "futures-core", "futures-util", - "mio 0.8.5", + "mio 0.8.9", "num_cpus", "socket2 0.4.7", - "tokio 1.25.0", + "tokio 1.34.0", "tracing", ] @@ -361,7 +361,7 @@ checksum = "8d3dc6a618b082974a08d7a4781d24d4691cba51500059bfebe6656a61ebfe1e" dependencies = [ "futures-core", "paste", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", ] [[package]] @@ -432,7 +432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" dependencies = [ "local-waker", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", ] [[package]] @@ -503,7 +503,7 @@ dependencies = [ "log", "mime", "once_cell", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "regex", "serde", "serde_json", @@ -527,8 +527,8 @@ dependencies = [ "bytes 1.3.0", "bytestring", "futures-core", - "pin-project-lite 0.2.9", - "tokio 1.25.0", + "pin-project-lite 0.2.13", + "tokio 1.34.0", "tokio-util 0.7.4", ] @@ -587,6 +587,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -687,7 +696,7 @@ version = "0.0.1" dependencies = [ "anyhow", "chrono", - "clap", + "clap 2.34.0", "config", "container_common", "dotenv", @@ -705,7 +714,7 @@ dependencies = [ "serde_json", "sqlx", "tokens", - "tokio 1.25.0", + "tokio 1.34.0", ] [[package]] @@ -717,6 +726,54 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "anstream" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + [[package]] name = "any_ascii" version = "0.1.7" @@ -775,8 +832,8 @@ dependencies = [ "flate2", "futures-core", "memchr", - "pin-project-lite 0.2.9", - "tokio 1.25.0", + "pin-project-lite 0.2.13", + "tokio 1.34.0", ] [[package]] @@ -800,7 +857,7 @@ dependencies = [ "smart-default", "smol_str", "thiserror", - "tokio 1.25.0", + "tokio 1.34.0", "uuid 0.8.2", ] @@ -918,6 +975,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.8" @@ -991,7 +1063,7 @@ dependencies = [ "serde_derive", "serde_json", "sqlx", - "tokio 1.25.0", + "tokio 1.34.0", "toml", "url_config", ] @@ -1289,6 +1361,46 @@ dependencies = [ "vec_map", ] +[[package]] +name = "clap" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.10.0", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck 0.4.0", + "proc-macro2", + "quote", + "syn 2.0.26", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + [[package]] name = "cloud_storage" version = "0.0.1" @@ -1298,7 +1410,7 @@ dependencies = [ "container_common", "log", "rust-s3", - "tokio 1.25.0", + "tokio 1.34.0", ] [[package]] @@ -1316,6 +1428,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "combine" version = "3.8.1" @@ -1338,8 +1456,8 @@ dependencies = [ "bytes 1.3.0", "futures-core", "memchr", - "pin-project-lite 0.2.9", - "tokio 1.25.0", + "pin-project-lite 0.2.13", + "tokio 1.34.0", "tokio-util 0.6.9", ] @@ -1772,7 +1890,7 @@ version = "0.0.1" dependencies = [ "anyhow", "chrono", - "clap", + "clap 2.34.0", "config", "container_common", "dotenv", @@ -1886,7 +2004,7 @@ name = "dev-database-seed" version = "0.0.1" dependencies = [ "buckets", - "clap", + "clap 2.34.0", "cloud_storage", "config", "easyenv", @@ -1902,7 +2020,7 @@ dependencies = [ "sqlx", "storyteller_root", "tokens", - "tokio 1.25.0", + "tokio 1.34.0", ] [[package]] @@ -1910,7 +2028,7 @@ name = "dev-upload-media-file" version = "0.0.1" dependencies = [ "buckets", - "clap", + "clap 2.34.0", "cloud_storage", "config", "easyenv", @@ -1923,7 +2041,7 @@ dependencies = [ "mysql_queries", "sqlx", "tokens", - "tokio 1.25.0", + "tokio 1.34.0", ] [[package]] @@ -2064,7 +2182,7 @@ dependencies = [ "anyhow", "bootstrap", "chrono", - "clap", + "clap 2.34.0", "cloud_storage", "concurrency", "config", @@ -2094,7 +2212,7 @@ dependencies = [ "subprocess", "subprocess_common", "tempdir", - "tokio 1.25.0", + "tokio 1.34.0", "tts_common", "zip", ] @@ -2142,7 +2260,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "tokio 1.25.0", + "tokio 1.34.0", "toml", ] @@ -2189,7 +2307,7 @@ dependencies = [ "lazy_static", "percent-encoding", "reqwest", - "rustc_version", + "rustc_version 0.2.3", "serde", "serde_json", "serde_with", @@ -2202,7 +2320,7 @@ name = "elasticsearch-cli" version = "0.0.1" dependencies = [ "buckets", - "clap", + "clap 2.34.0", "cloud_storage", "config", "easyenv", @@ -2223,7 +2341,7 @@ dependencies = [ "storyteller_root", "strum", "tokens", - "tokio 1.25.0", + "tokio 1.34.0", ] [[package]] @@ -2552,7 +2670,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "waker-fn", ] @@ -2598,7 +2716,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "pin-utils", "slab", ] @@ -2665,6 +2783,12 @@ dependencies = [ "weezl", ] +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "google_drive_common" version = "0.0.1" @@ -2710,7 +2834,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-util 0.7.4", "tracing", ] @@ -2855,7 +2979,7 @@ checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" dependencies = [ "bytes 1.3.0", "http", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", ] [[package]] @@ -2876,7 +3000,7 @@ dependencies = [ "futures-lite", "http", "infer 0.2.3", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "rand 0.7.3", "serde", "serde_json", @@ -2954,9 +3078,9 @@ dependencies = [ "httparse", "httpdate", "itoa 1.0.3", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "socket2 0.4.7", - "tokio 1.25.0", + "tokio 1.34.0", "tower-service", "tracing", "want", @@ -2971,7 +3095,7 @@ dependencies = [ "bytes 1.3.0", "hyper", "native-tls", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-native-tls", ] @@ -3083,7 +3207,7 @@ dependencies = [ "bootstrap", "buckets", "chrono", - "clap", + "clap 2.34.0", "cloud_storage", "concurrency", "config", @@ -3117,7 +3241,7 @@ dependencies = [ "tempdir", "testing", "tokens", - "tokio 1.25.0", + "tokio 1.34.0", "tts_common", ] @@ -3346,9 +3470,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.141" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libm" @@ -3442,6 +3566,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "logging" +version = "0.0.1" +dependencies = [ + "dotenv", + "env_logger 0.10.0", + "log", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -3451,6 +3584,20 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lru-delete-models" +version = "0.0.1" +dependencies = [ + "clap 4.4.7", + "easyenv", + "errors", + "filesys", + "logging", + "storyteller_root", + "tokio 1.34.0", + "walkdir", +] + [[package]] name = "lru_time_cache" version = "0.11.11" @@ -3648,14 +3795,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -3808,7 +3955,7 @@ dependencies = [ "rand 0.7.3", "serde", "serde_json", - "tokio 1.25.0", + "tokio 1.34.0", "uuid 0.8.2", ] @@ -3965,6 +4112,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -4267,9 +4423,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -4630,9 +4786,9 @@ dependencies = [ "futures-util", "itoa 0.4.7", "percent-encoding", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "sha1 0.6.0", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-util 0.6.9", "url", ] @@ -4747,11 +4903,11 @@ dependencies = [ "native-tls", "once_cell", "percent-encoding", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "serde", "serde_json", "serde_urlencoded", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-native-tls", "tokio-util 0.7.4", "tower-service", @@ -4907,11 +5063,17 @@ dependencies = [ "sha2 0.10.5", "thiserror", "time 0.3.14", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-stream", "url", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-serialize" version = "0.3.24" @@ -4988,10 +5150,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c7c4cd1e0a04c48f953473383a60143884515b7a8eb7ca7d9b1baa9c05dee75" dependencies = [ "bytes 1.3.0", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "rxml_validation", "smartstring", - "tokio 1.25.0", + "tokio 1.34.0", ] [[package]] @@ -5006,6 +5168,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.19" @@ -5035,9 +5206,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" name = "scrape_test" version = "0.0.1" dependencies = [ - "clap", + "clap 2.34.0", "errors", - "tokio 1.25.0", + "tokio 1.34.0", "web_scrapers", ] @@ -5429,6 +5600,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "speculoos" version = "0.9.0" @@ -5565,7 +5746,7 @@ checksum = "24c5b2d25fa654cc5f841750b8e1cdedbe21189bf9a9382ee90bfa9dd3562396" dependencies = [ "native-tls", "once_cell", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-native-tls", ] @@ -5714,7 +5895,7 @@ dependencies = [ "native-tls", "newrelic-telemetry", "once_cell", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "pulldown-cmark", "r2d2_redis", "rand 0.8.5", @@ -5736,7 +5917,7 @@ dependencies = [ "tempdir", "time 0.2.26", "tokens", - "tokio 1.25.0", + "tokio 1.34.0", "toml", "tts_common", "twitch_api2", @@ -6328,22 +6509,21 @@ dependencies = [ [[package]] name = "tokio" -version = "1.25.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ - "autocfg", + "backtrace", "bytes 1.3.0", "libc", - "memchr", - "mio 0.8.5", + "mio 0.8.9", "num_cpus", "parking_lot 0.12.1", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "signal-hook-registry", - "socket2 0.4.7", + "socket2 0.5.5", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -6380,13 +6560,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.26", ] [[package]] @@ -6396,7 +6576,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.25.0", + "tokio 1.34.0", ] [[package]] @@ -6425,7 +6605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ "rustls", - "tokio 1.25.0", + "tokio 1.34.0", "webpki", ] @@ -6436,8 +6616,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", - "pin-project-lite 0.2.9", - "tokio 1.25.0", + "pin-project-lite 0.2.13", + "tokio 1.34.0", ] [[package]] @@ -6474,7 +6654,7 @@ dependencies = [ "log", "pin-project 1.0.7", "rustls", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-rustls", "tungstenite", "webpki", @@ -6523,8 +6703,8 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.9", - "tokio 1.25.0", + "pin-project-lite 0.2.13", + "tokio 1.34.0", ] [[package]] @@ -6536,8 +6716,8 @@ dependencies = [ "bytes 1.3.0", "futures-core", "futures-sink", - "pin-project-lite 0.2.9", - "tokio 1.25.0", + "pin-project-lite 0.2.13", + "tokio 1.34.0", "tracing", ] @@ -6564,7 +6744,7 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.13", "tracing-core", ] @@ -6638,7 +6818,7 @@ version = "0.0.1" dependencies = [ "anyhow", "chrono", - "clap", + "clap 2.34.0", "cloud_storage", "config", "container_common", @@ -6661,7 +6841,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.25.0", + "tokio 1.34.0", "tts_common", ] @@ -6671,7 +6851,7 @@ version = "0.0.1" dependencies = [ "anyhow", "chrono", - "clap", + "clap 2.34.0", "cloud_storage", "config", "container_common", @@ -6693,7 +6873,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.25.0", + "tokio 1.34.0", "tts_common", ] @@ -6792,7 +6972,7 @@ dependencies = [ "sqlx", "time 0.2.26", "tokens", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-tungstenite", "toml", "tts_common", @@ -7039,6 +7219,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "uuid" version = "0.8.2" @@ -7085,7 +7271,7 @@ dependencies = [ "anyhow", "buckets", "chrono", - "clap", + "clap 2.34.0", "cloud_storage", "config", "container_common", @@ -7107,7 +7293,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.25.0", + "tokio 1.34.0", "tts_common", ] @@ -7117,7 +7303,7 @@ version = "0.0.1" dependencies = [ "anyhow", "chrono", - "clap", + "clap 2.34.0", "cloud_storage", "config", "container_common", @@ -7138,7 +7324,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.25.0", + "tokio 1.34.0", "tts_common", ] @@ -7148,6 +7334,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" @@ -7362,7 +7558,7 @@ dependencies = [ "sha2 0.9.5", "sqlx", "time 0.2.26", - "tokio 1.25.0", + "tokio 1.34.0", "tokio-tungstenite", "toml", "twitch_api2", @@ -7426,21 +7622,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.1", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", - "windows_x86_64_gnullvm 0.42.1", - "windows_x86_64_msvc 0.42.1", -] - [[package]] name = "windows-sys" version = "0.45.0" diff --git a/Cargo.toml b/Cargo.toml index e6576178c9..6470185b75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/cli/dev_upload_media_file", "crates/cli/elasticsearch-cli", "crates/cli/kubectl-pod-cleanup", + "crates/cli/lru-delete-models", "crates/cli/scrape_test", # Shared libraries @@ -43,6 +44,7 @@ members = [ "crates/lib/files/mimetypes", "crates/lib/idempotency", "crates/lib/jobs_common", + "crates/lib/logging", "crates/lib/mysql_queries", "crates/lib/redis_common", "crates/lib/server_environment", diff --git a/crates/cli/kubectl-pod-cleanup/src/main.rs b/crates/cli/kubectl-pod-cleanup/src/main.rs index d97961dad3..0f2fb3cab9 100644 --- a/crates/cli/kubectl-pod-cleanup/src/main.rs +++ b/crates/cli/kubectl-pod-cleanup/src/main.rs @@ -1,3 +1,20 @@ +//! kube-pod-cleanup +//! +//! Rationale: In production, dead/evicted pods stick around for eternity. +//! Within a matter of days, thousands of dead pods can clutter the output of `kubectl`, +//! making it operationally difficult to maintain the cluster. +//! +//! This tool quickly dispenses of garbage pods. (Though we may still want to keep them +//! for debugging actual issues.) +//! +//! To install as a system binary, do the following: +//! +//! cargo install --path ./crates/cli/kubectl_pod_cleanup --bin kubectl-pod-cleanup +//! +//! (Cargo install seems to currently require the --path argument in a workspace, +//! as tracked by this issue: https://stackoverflow.com/a/76271890 ) +//! + use std::thread; use std::time::Duration; @@ -13,23 +30,6 @@ pub mod delete_pods_threaded; pub mod list_pods; pub mod pod_store; -/// kube-pod-cleanup -/// -/// Rationale: In production, dead/evicted pods stick around for eternity. -/// Within a matter of days, thousands of dead pods can clutter the output of `kubectl`, -/// making it operationally difficult to maintain the cluster. -/// -/// This tool quickly dispenses of garbage pods. (Though we may still want to keep them -/// for debugging actual issues.) -/// -/// To install as a system binary, do the following: -/// -/// cargo install --path ./crates/cli/kubectl_pod_cleanup --bin kubectl-pod-cleanup -/// -/// (Cargo install seems to currently require the --path argument in a workspace, -/// as tracked by this issue: https://stackoverflow.com/a/76271890 ) -/// - pub fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); diff --git a/crates/cli/lru-delete-models/Cargo.toml b/crates/cli/lru-delete-models/Cargo.toml new file mode 100644 index 0000000000..81a24f304d --- /dev/null +++ b/crates/cli/lru-delete-models/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "lru-delete-models" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "lru-delete-models" +path = "src/main.rs" + +[dependencies] +# Internal +easyenv = { path = "../../lib/easyenv" } +errors = { path = "../../lib/errors" } +filesys = { path = "../../lib/files/filesys" } +logging = { path = "../../lib/logging" } +storyteller_root = { path = "../../lib/storyteller_root" } + +# External +clap = { version = "4.4.7", features = ["derive"] } +walkdir = "2.4.0" + +# Runtime +tokio = { version = "1.34.0", features = ["macros"] } + +[dev-dependencies] +# None yet diff --git a/crates/cli/lru-delete-models/src/main.rs b/crates/cli/lru-delete-models/src/main.rs new file mode 100644 index 0000000000..ce3cf1417c --- /dev/null +++ b/crates/cli/lru-delete-models/src/main.rs @@ -0,0 +1,124 @@ +//! lru-delete-models +//! +//! Amazon EFS is costing us a ton of money and we need a quick way to stop the bleeding. +//! + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::thread; +use std::time::{Duration, SystemTime}; + +use clap::Parser; +use walkdir::WalkDir; + +use errors::AnyhowResult; +use logging::{error, info, init_env_logger, TYPICAL_LOG_LEVEL}; + +/// TTS models start with this prefix, eg. "TM:r60rp93g7d1x" +const TTS_MODEL_PREFIX : &str = "TM:"; + +/// VM models start with this prefix, eg. "VM:8jbc6dh0b77d" +const VOCODER_MODEL_PREFIX : &str = "VM:"; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct CliArgs { + // Path to TTS models and vocoder models to clear + // The script will recursively scan the directories. + #[arg(short, long)] + directory: String, + + // Number of models to keep + #[arg(short, long, default_value_t = 100)] + keep: usize, + + // Only perform the deletes when this is set. + #[arg(short, long, default_value_t = false)] + confirm: bool, +} + +#[derive(Debug)] +struct FileData { + path: PathBuf, + last_access: SystemTime, +} + +pub fn main() -> AnyhowResult<()> { + init_env_logger(Some(TYPICAL_LOG_LEVEL)); + + info!("lru-delete-models"); + + let args = CliArgs::parse(); + + delete_files_loop(args); + + Ok(()) +} + +fn delete_files_loop(args: CliArgs) { + loop { + let files = find_model_files(&args.directory); + let total_file_count = files.len(); + + let files = scope_to_files_to_remove(files, args.keep); + + info!("Files total: {total_file_count} ; to delete: {}", files.len()); + + for file in files { + if !args.confirm { + info!("would have removed file (use --confirm to delete): {:?}", file); + } else { + info!("removing file: {:?}", file); + + if let Err(err) = std::fs::remove_file(file) { + error!("Error removing file: {err}"); + } + } + + thread::sleep(Duration::from_millis(100)); + } + + thread::sleep(Duration::from_secs(10)); + } +} + +fn find_model_files(directory: &str) -> Vec { + WalkDir::new(directory).into_iter() + .filter_map(Result::ok) + .filter(|e| !e.file_type().is_dir()) + .map(|e| e.path().canonicalize()) + .filter_map(Result::ok) + .filter(|path| { + path.file_name() + .and_then(|f| f.to_str()) + .map(|f| f.starts_with(TTS_MODEL_PREFIX) || f.starts_with(VOCODER_MODEL_PREFIX)) + //.map(|f| f.ends_with(".jpg") || f.starts_with("README")) + .unwrap_or(false) + }) + .collect::>() +} + +fn scope_to_files_to_remove(files: Vec, keep_count: usize) -> Vec { + let mut files = file_metadata_map(files); + let mut files_to_remove = Vec::with_capacity(files.len()); + + while files.len() > keep_count { + if let Some((_time, file_data)) = files.pop_first() { + files_to_remove.push(file_data.path); + } + } + + files_to_remove +} + +fn file_metadata_map(files: Vec) -> BTreeMap { + files.into_iter() + .map(|path| FileData { + last_access: path.metadata() + .and_then(|m| m.accessed()) + .unwrap_or(SystemTime::UNIX_EPOCH), + path, + }) + .map(|file_data| (file_data.last_access, file_data)) + .collect::>() +} diff --git a/crates/lib/logging/Cargo.toml b/crates/lib/logging/Cargo.toml new file mode 100644 index 0000000000..71de22e8ab --- /dev/null +++ b/crates/lib/logging/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "logging" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +# NB: Public "logging" crate is five years old and uninteresting. Stealing this name for internal use. +name = "logging" +path = "src/lib.rs" + +[dependencies] +dotenv = "0.15.0" +env_logger = "0.10.0" +log = "0.4.20" + +[dev-dependencies] +# None yet diff --git a/crates/lib/logging/src/lib.rs b/crates/lib/logging/src/lib.rs new file mode 100644 index 0000000000..b0311e36bf --- /dev/null +++ b/crates/lib/logging/src/lib.rs @@ -0,0 +1,72 @@ +//! logging +//! +//! The purpose of this crate is to pin against a single version of the 'log' crate, making it +//! easier to simultaneously update across apps. We can also pack in a few useful definitions. +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +use std::env; + +/// Re-export. +pub use log::debug; +pub use log::error; +pub use log::info; +pub use log::log; +pub use log::trace; +pub use log::warn; + +/// Name of the environment variable Rust's env logger uses +pub const ENV_RUST_LOG : &str = "RUST_LOG"; + +/// The default logging level +pub const DEFAULT_LOG_LEVEL: &str = "info"; + +/// A useful logging level +/// NB: `sqlx::query` is spammy and dumps all queries as "info"-level log lines. +/// NB: `hyper::proto::h1::io` is incredibly spammy and logs every chunk of bytes in very large files being downloaded. +pub const TYPICAL_LOG_LEVEL : &str = concat!( + "debug,", + "actix_web=info,", + "sqlx::query=warn,", + "hyper::proto::h1::io=warn,", + "storyteller_web::threads::db_health_checker_thread::db_health_checker_thread=warn,", + "http_server_common::request::get_request_ip=info," // Debug spams Rust logs +); + +/// Initialize Rust's env logger. +/// +/// The Rust logger reads the desired log level from the `RUST_LOG` environment variable. If this +/// isn't set, the provided default is used. If a default fallback isn't provided to this function, +/// we fall back to `"info"`. +/// +/// A more robust logging config might configure on a per-component basis, eg. +/// `"tokio_reactor=warn,hyper=info,debug"`. You can read more in the `log` and `env_logger` crate +/// docs. +pub fn init_env_logger(default_if_absent: Option<&str>) { + if env::var(ENV_RUST_LOG) + .as_ref() + .ok() + .is_none() + { + let default_log_level = default_if_absent.unwrap_or(DEFAULT_LOG_LEVEL); + println!("Setting default logging level to \"{}\", override with env var {}.", + default_log_level, ENV_RUST_LOG); + env::set_var(ENV_RUST_LOG, default_log_level); + } + + env_logger::init(); +} +