gsadmin/artcraft
1
0
Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-10-26 11:09:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Delete models from AWS EFS host

Browse Source
This commit is contained in:
Brandon Thomas
2023-11-09 23:13:08 -05:00
parent a34785c308
commit fc344557be
8 changed files with 561 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/dictionaries/bt.xml
Generated
+1
View File
@@ -33,6 +33,7 @@
<w>retryably</w>
<w>scrollback</w>
<w>softvc</w>
<w>spammy</w>
<w>sql's</w>
<w>symphonia</w>
<w>tacotron</w>
Cargo.lock
Generated
+292 -111
View File
@@ -20,9 +20,9 @@ dependencies = [
"log",
"once_cell",
"parking_lot 0.11.1",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"smallvec 1.10.0",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-util 0.6.9",
]
@@ -53,8 +53,8 @@ dependencies = [
"futures-core",
"futures-sink",
"memchr",
"pin-project-lite 0.2.9",
"tokio 1.25.0",
"pin-project-lite 0.2.13",
"tokio 1.34.0",
"tokio-util 0.7.4",
"tracing",
]
@@ -127,7 +127,7 @@ dependencies = [
"mime",
"mime_guess",
"percent-encoding",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
]
[[package]]
@@ -204,11 +204,11 @@ dependencies = [
"local-channel",
"mime",
"percent-encoding",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"rand 0.8.5",
"sha1 0.10.4",
"smallvec 1.10.0",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-util 0.7.4",
"tracing",
"zstd 0.12.3+zstd.1.5.2",
@@ -302,7 +302,7 @@ checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d"
dependencies = [
"actix-macros 0.2.3",
"futures-core",
"tokio 1.25.0",
"tokio 1.34.0",
]
[[package]]
@@ -336,10 +336,10 @@ dependencies = [
"actix-utils 3.0.0",
"futures-core",
"futures-util",
"mio 0.8.5",
"mio 0.8.9",
"num_cpus",
"socket2 0.4.7",
"tokio 1.25.0",
"tokio 1.34.0",
"tracing",
]
@@ -361,7 +361,7 @@ checksum = "8d3dc6a618b082974a08d7a4781d24d4691cba51500059bfebe6656a61ebfe1e"
dependencies = [
"futures-core",
"paste",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
]
[[package]]
@@ -432,7 +432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94"
dependencies = [
"local-waker",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
]
[[package]]
@@ -503,7 +503,7 @@ dependencies = [
"log",
"mime",
"once_cell",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"regex",
"serde",
"serde_json",
@@ -527,8 +527,8 @@ dependencies = [
"bytes 1.3.0",
"bytestring",
"futures-core",
"pin-project-lite 0.2.9",
"tokio 1.25.0",
"pin-project-lite 0.2.13",
"tokio 1.34.0",
"tokio-util 0.7.4",
]
@@ -587,6 +587,15 @@ dependencies = [
"serde_json",
]
[[package]]
name = "addr2line"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "1.0.2"
@@ -687,7 +696,7 @@ version = "0.0.1"
dependencies = [
"anyhow",
"chrono",
"clap",
"clap 2.34.0",
"config",
"container_common",
"dotenv",
@@ -705,7 +714,7 @@ dependencies = [
"serde_json",
"sqlx",
"tokens",
"tokio 1.25.0",
"tokio 1.34.0",
]
[[package]]
@@ -717,6 +726,54 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "anstream"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
[[package]]
name = "anstyle-parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628"
dependencies = [
"anstyle",
"windows-sys 0.48.0",
]
[[package]]
name = "any_ascii"
version = "0.1.7"
@@ -775,8 +832,8 @@ dependencies = [
"flate2",
"futures-core",
"memchr",
"pin-project-lite 0.2.9",
"tokio 1.25.0",
"pin-project-lite 0.2.13",
"tokio 1.34.0",
]
[[package]]
@@ -800,7 +857,7 @@ dependencies = [
"smart-default",
"smol_str",
"thiserror",
"tokio 1.25.0",
"tokio 1.34.0",
"uuid 0.8.2",
]
@@ -918,6 +975,21 @@ dependencies = [
"thiserror",
]
[[package]]
name = "backtrace"
version = "0.3.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
dependencies = [
"addr2line",
"cc",
"cfg-if 1.0.0",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "base-x"
version = "0.2.8"
@@ -991,7 +1063,7 @@ dependencies = [
"serde_derive",
"serde_json",
"sqlx",
"tokio 1.25.0",
"tokio 1.34.0",
"toml",
"url_config",
]
@@ -1289,6 +1361,46 @@ dependencies = [
"vec_map",
]
[[package]]
name = "clap"
version = "4.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b"
dependencies = [
"clap_builder",
"clap_derive",
]
[[package]]
name = "clap_builder"
version = "4.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim 0.10.0",
]
[[package]]
name = "clap_derive"
version = "4.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
dependencies = [
"heck 0.4.0",
"proc-macro2",
"quote",
"syn 2.0.26",
]
[[package]]
name = "clap_lex"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
[[package]]
name = "cloud_storage"
version = "0.0.1"
@@ -1298,7 +1410,7 @@ dependencies = [
"container_common",
"log",
"rust-s3",
"tokio 1.25.0",
"tokio 1.34.0",
]
[[package]]
@@ -1316,6 +1428,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "colorchoice"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "combine"
version = "3.8.1"
@@ -1338,8 +1456,8 @@ dependencies = [
"bytes 1.3.0",
"futures-core",
"memchr",
"pin-project-lite 0.2.9",
"tokio 1.25.0",
"pin-project-lite 0.2.13",
"tokio 1.34.0",
"tokio-util 0.6.9",
]
@@ -1772,7 +1890,7 @@ version = "0.0.1"
dependencies = [
"anyhow",
"chrono",
"clap",
"clap 2.34.0",
"config",
"container_common",
"dotenv",
@@ -1886,7 +2004,7 @@ name = "dev-database-seed"
version = "0.0.1"
dependencies = [
"buckets",
"clap",
"clap 2.34.0",
"cloud_storage",
"config",
"easyenv",
@@ -1902,7 +2020,7 @@ dependencies = [
"sqlx",
"storyteller_root",
"tokens",
"tokio 1.25.0",
"tokio 1.34.0",
]
[[package]]
@@ -1910,7 +2028,7 @@ name = "dev-upload-media-file"
version = "0.0.1"
dependencies = [
"buckets",
"clap",
"clap 2.34.0",
"cloud_storage",
"config",
"easyenv",
@@ -1923,7 +2041,7 @@ dependencies = [
"mysql_queries",
"sqlx",
"tokens",
"tokio 1.25.0",
"tokio 1.34.0",
]
[[package]]
@@ -2064,7 +2182,7 @@ dependencies = [
"anyhow",
"bootstrap",
"chrono",
"clap",
"clap 2.34.0",
"cloud_storage",
"concurrency",
"config",
@@ -2094,7 +2212,7 @@ dependencies = [
"subprocess",
"subprocess_common",
"tempdir",
"tokio 1.25.0",
"tokio 1.34.0",
"tts_common",
"zip",
]
@@ -2142,7 +2260,7 @@ dependencies = [
"serde",
"serde_derive",
"serde_json",
"tokio 1.25.0",
"tokio 1.34.0",
"toml",
]
@@ -2189,7 +2307,7 @@ dependencies = [
"lazy_static",
"percent-encoding",
"reqwest",
"rustc_version",
"rustc_version 0.2.3",
"serde",
"serde_json",
"serde_with",
@@ -2202,7 +2320,7 @@ name = "elasticsearch-cli"
version = "0.0.1"
dependencies = [
"buckets",
"clap",
"clap 2.34.0",
"cloud_storage",
"config",
"easyenv",
@@ -2223,7 +2341,7 @@ dependencies = [
"storyteller_root",
"strum",
"tokens",
"tokio 1.25.0",
"tokio 1.34.0",
]
[[package]]
@@ -2552,7 +2670,7 @@ dependencies = [
"futures-io",
"memchr",
"parking",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"waker-fn",
]
@@ -2598,7 +2716,7 @@ dependencies = [
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"pin-utils",
"slab",
]
@@ -2665,6 +2783,12 @@ dependencies = [
"weezl",
]
[[package]]
name = "gimli"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
[[package]]
name = "google_drive_common"
version = "0.0.1"
@@ -2710,7 +2834,7 @@ dependencies = [
"http",
"indexmap",
"slab",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-util 0.7.4",
"tracing",
]
@@ -2855,7 +2979,7 @@ checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
dependencies = [
"bytes 1.3.0",
"http",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
]
[[package]]
@@ -2876,7 +3000,7 @@ dependencies = [
"futures-lite",
"http",
"infer 0.2.3",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"rand 0.7.3",
"serde",
"serde_json",
@@ -2954,9 +3078,9 @@ dependencies = [
"httparse",
"httpdate",
"itoa 1.0.3",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"socket2 0.4.7",
"tokio 1.25.0",
"tokio 1.34.0",
"tower-service",
"tracing",
"want",
@@ -2971,7 +3095,7 @@ dependencies = [
"bytes 1.3.0",
"hyper",
"native-tls",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-native-tls",
]
@@ -3083,7 +3207,7 @@ dependencies = [
"bootstrap",
"buckets",
"chrono",
"clap",
"clap 2.34.0",
"cloud_storage",
"concurrency",
"config",
@@ -3117,7 +3241,7 @@ dependencies = [
"tempdir",
"testing",
"tokens",
"tokio 1.25.0",
"tokio 1.34.0",
"tts_common",
]
@@ -3346,9 +3470,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.141"
version = "0.2.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
[[package]]
name = "libm"
@@ -3442,6 +3566,15 @@ version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "logging"
version = "0.0.1"
dependencies = [
"dotenv",
"env_logger 0.10.0",
"log",
]
[[package]]
name = "lru-cache"
version = "0.1.2"
@@ -3451,6 +3584,20 @@ dependencies = [
"linked-hash-map",
]
[[package]]
name = "lru-delete-models"
version = "0.0.1"
dependencies = [
"clap 4.4.7",
"easyenv",
"errors",
"filesys",
"logging",
"storyteller_root",
"tokio 1.34.0",
"walkdir",
]
[[package]]
name = "lru_time_cache"
version = "0.11.11"
@@ -3648,14 +3795,14 @@ dependencies = [
[[package]]
name = "mio"
version = "0.8.5"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0"
dependencies = [
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.42.0",
"windows-sys 0.48.0",
]
[[package]]
@@ -3808,7 +3955,7 @@ dependencies = [
"rand 0.7.3",
"serde",
"serde_json",
"tokio 1.25.0",
"tokio 1.34.0",
"uuid 0.8.2",
]
@@ -3965,6 +4112,15 @@ dependencies = [
"libc",
]
[[package]]
name = "object"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.18.0"
@@ -4267,9 +4423,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777"
[[package]]
name = "pin-project-lite"
version = "0.2.9"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
[[package]]
name = "pin-utils"
@@ -4630,9 +4786,9 @@ dependencies = [
"futures-util",
"itoa 0.4.7",
"percent-encoding",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"sha1 0.6.0",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-util 0.6.9",
"url",
]
@@ -4747,11 +4903,11 @@ dependencies = [
"native-tls",
"once_cell",
"percent-encoding",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"serde",
"serde_json",
"serde_urlencoded",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-native-tls",
"tokio-util 0.7.4",
"tower-service",
@@ -4907,11 +5063,17 @@ dependencies = [
"sha2 0.10.5",
"thiserror",
"time 0.3.14",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-stream",
"url",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustc-serialize"
version = "0.3.24"
@@ -4988,10 +5150,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c7c4cd1e0a04c48f953473383a60143884515b7a8eb7ca7d9b1baa9c05dee75"
dependencies = [
"bytes 1.3.0",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"rxml_validation",
"smartstring",
"tokio 1.25.0",
"tokio 1.34.0",
]
[[package]]
@@ -5006,6 +5168,15 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.19"
@@ -5035,9 +5206,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
name = "scrape_test"
version = "0.0.1"
dependencies = [
"clap",
"clap 2.34.0",
"errors",
"tokio 1.25.0",
"tokio 1.34.0",
"web_scrapers",
]
@@ -5429,6 +5600,16 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "socket2"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
dependencies = [
"libc",
"windows-sys 0.48.0",
]
[[package]]
name = "speculoos"
version = "0.9.0"
@@ -5565,7 +5746,7 @@ checksum = "24c5b2d25fa654cc5f841750b8e1cdedbe21189bf9a9382ee90bfa9dd3562396"
dependencies = [
"native-tls",
"once_cell",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-native-tls",
]
@@ -5714,7 +5895,7 @@ dependencies = [
"native-tls",
"newrelic-telemetry",
"once_cell",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"pulldown-cmark",
"r2d2_redis",
"rand 0.8.5",
@@ -5736,7 +5917,7 @@ dependencies = [
"tempdir",
"time 0.2.26",
"tokens",
"tokio 1.25.0",
"tokio 1.34.0",
"toml",
"tts_common",
"twitch_api2",
@@ -6328,22 +6509,21 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.25.0"
version = "1.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"
checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
dependencies = [
"autocfg",
"backtrace",
"bytes 1.3.0",
"libc",
"memchr",
"mio 0.8.5",
"mio 0.8.9",
"num_cpus",
"parking_lot 0.12.1",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"signal-hook-registry",
"socket2 0.4.7",
"socket2 0.5.5",
"tokio-macros",
"windows-sys 0.42.0",
"windows-sys 0.48.0",
]
[[package]]
@@ -6380,13 +6560,13 @@ dependencies = [
[[package]]
name = "tokio-macros"
version = "1.8.2"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.107",
"syn 2.0.26",
]
[[package]]
@@ -6396,7 +6576,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
"native-tls",
"tokio 1.25.0",
"tokio 1.34.0",
]
[[package]]
@@ -6425,7 +6605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
dependencies = [
"rustls",
"tokio 1.25.0",
"tokio 1.34.0",
"webpki",
]
@@ -6436,8 +6616,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce"
dependencies = [
"futures-core",
"pin-project-lite 0.2.9",
"tokio 1.25.0",
"pin-project-lite 0.2.13",
"tokio 1.34.0",
]
[[package]]
@@ -6474,7 +6654,7 @@ dependencies = [
"log",
"pin-project 1.0.7",
"rustls",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-rustls",
"tungstenite",
"webpki",
@@ -6523,8 +6703,8 @@ dependencies = [
"futures-core",
"futures-sink",
"log",
"pin-project-lite 0.2.9",
"tokio 1.25.0",
"pin-project-lite 0.2.13",
"tokio 1.34.0",
]
[[package]]
@@ -6536,8 +6716,8 @@ dependencies = [
"bytes 1.3.0",
"futures-core",
"futures-sink",
"pin-project-lite 0.2.9",
"tokio 1.25.0",
"pin-project-lite 0.2.13",
"tokio 1.34.0",
"tracing",
]
@@ -6564,7 +6744,7 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
dependencies = [
"cfg-if 1.0.0",
"log",
"pin-project-lite 0.2.9",
"pin-project-lite 0.2.13",
"tracing-core",
]
@@ -6638,7 +6818,7 @@ version = "0.0.1"
dependencies = [
"anyhow",
"chrono",
"clap",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
@@ -6661,7 +6841,7 @@ dependencies = [
"sqlx",
"subprocess",
"tempdir",
"tokio 1.25.0",
"tokio 1.34.0",
"tts_common",
]
@@ -6671,7 +6851,7 @@ version = "0.0.1"
dependencies = [
"anyhow",
"chrono",
"clap",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
@@ -6693,7 +6873,7 @@ dependencies = [
"sqlx",
"subprocess",
"tempdir",
"tokio 1.25.0",
"tokio 1.34.0",
"tts_common",
]
@@ -6792,7 +6972,7 @@ dependencies = [
"sqlx",
"time 0.2.26",
"tokens",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-tungstenite",
"toml",
"tts_common",
@@ -7039,6 +7219,12 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "utf8parse"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "uuid"
version = "0.8.2"
@@ -7085,7 +7271,7 @@ dependencies = [
"anyhow",
"buckets",
"chrono",
"clap",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
@@ -7107,7 +7293,7 @@ dependencies = [
"sqlx",
"subprocess",
"tempdir",
"tokio 1.25.0",
"tokio 1.34.0",
"tts_common",
]
@@ -7117,7 +7303,7 @@ version = "0.0.1"
dependencies = [
"anyhow",
"chrono",
"clap",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
@@ -7138,7 +7324,7 @@ dependencies = [
"sqlx",
"subprocess",
"tempdir",
"tokio 1.25.0",
"tokio 1.34.0",
"tts_common",
]
@@ -7148,6 +7334,16 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
[[package]]
name = "walkdir"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.0"
@@ -7362,7 +7558,7 @@ dependencies = [
"sha2 0.9.5",
"sqlx",
"time 0.2.26",
"tokio 1.25.0",
"tokio 1.34.0",
"tokio-tungstenite",
"toml",
"twitch_api2",
@@ -7426,21 +7622,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-sys"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
dependencies = [
"windows_aarch64_gnullvm 0.42.1",
"windows_aarch64_msvc 0.42.1",
"windows_i686_gnu 0.42.1",
"windows_i686_msvc 0.42.1",
"windows_x86_64_gnu 0.42.1",
"windows_x86_64_gnullvm 0.42.1",
"windows_x86_64_msvc 0.42.1",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
Cargo.toml
+2
View File
@@ -15,6 +15,7 @@ members = [
"crates/cli/dev_upload_media_file",
"crates/cli/elasticsearch-cli",
"crates/cli/kubectl-pod-cleanup",
"crates/cli/lru-delete-models",
"crates/cli/scrape_test",
# Shared libraries
@@ -43,6 +44,7 @@ members = [
"crates/lib/files/mimetypes",
"crates/lib/idempotency",
"crates/lib/jobs_common",
"crates/lib/logging",
"crates/lib/mysql_queries",
"crates/lib/redis_common",
"crates/lib/server_environment",
crates/cli/kubectl-pod-cleanup/src/main.rs
+17 -17
View File
@@ -1,3 +1,20 @@
//! kube-pod-cleanup
//!
//! Rationale: In production, dead/evicted pods stick around for eternity.
//! Within a matter of days, thousands of dead pods can clutter the output of `kubectl`,
//! making it operationally difficult to maintain the cluster.
//!
//! This tool quickly dispenses of garbage pods. (Though we may still want to keep them
//! for debugging actual issues.)
//!
//! To install as a system binary, do the following:
//!
//! cargo install --path ./crates/cli/kubectl_pod_cleanup --bin kubectl-pod-cleanup
//!
//! (Cargo install seems to currently require the --path argument in a workspace,
//! as tracked by this issue: https://stackoverflow.com/a/76271890 )
//!
use std::thread;
use std::time::Duration;
@@ -13,23 +30,6 @@ pub mod delete_pods_threaded;
pub mod list_pods;
pub mod pod_store;
/// kube-pod-cleanup
///
/// Rationale: In production, dead/evicted pods stick around for eternity.
/// Within a matter of days, thousands of dead pods can clutter the output of `kubectl`,
/// making it operationally difficult to maintain the cluster.
///
/// This tool quickly dispenses of garbage pods. (Though we may still want to keep them
/// for debugging actual issues.)
///
/// To install as a system binary, do the following:
///
/// cargo install --path ./crates/cli/kubectl_pod_cleanup --bin kubectl-pod-cleanup
///
/// (Cargo install seems to currently require the --path argument in a workspace,
/// as tracked by this issue: https://stackoverflow.com/a/76271890 )
///
pub fn main() -> AnyhowResult<()> {
easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
crates/cli/lru-delete-models/Cargo.toml
+31
View File
@@ -0,0 +1,31 @@
[package]
name = "lru-delete-models"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[[bin]]
name = "lru-delete-models"
path = "src/main.rs"
[dependencies]
# Internal
easyenv = { path = "../../lib/easyenv" }
errors = { path = "../../lib/errors" }
filesys = { path = "../../lib/files/filesys" }
logging = { path = "../../lib/logging" }
storyteller_root = { path = "../../lib/storyteller_root" }
# External
clap = { version = "4.4.7", features = ["derive"] }
walkdir = "2.4.0"
# Runtime
tokio = { version = "1.34.0", features = ["macros"] }
[dev-dependencies]
# None yet
crates/cli/lru-delete-models/src/main.rs
+124
View File
@@ -0,0 +1,124 @@
//! lru-delete-models
//!
//! Amazon EFS is costing us a ton of money and we need a quick way to stop the bleeding.
//!
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::thread;
use std::time::{Duration, SystemTime};
use clap::Parser;
use walkdir::WalkDir;
use errors::AnyhowResult;
use logging::{error, info, init_env_logger, TYPICAL_LOG_LEVEL};
/// TTS models start with this prefix, eg. "TM:r60rp93g7d1x"
const TTS_MODEL_PREFIX : &str = "TM:";
/// VM models start with this prefix, eg. "VM:8jbc6dh0b77d"
const VOCODER_MODEL_PREFIX : &str = "VM:";
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct CliArgs {
// Path to TTS models and vocoder models to clear
// The script will recursively scan the directories.
#[arg(short, long)]
directory: String,
// Number of models to keep
#[arg(short, long, default_value_t = 100)]
keep: usize,
// Only perform the deletes when this is set.
#[arg(short, long, default_value_t = false)]
confirm: bool,
}
#[derive(Debug)]
struct FileData {
path: PathBuf,
last_access: SystemTime,
}
pub fn main() -> AnyhowResult<()> {
init_env_logger(Some(TYPICAL_LOG_LEVEL));
info!("lru-delete-models");
let args = CliArgs::parse();
delete_files_loop(args);
Ok(())
}
fn delete_files_loop(args: CliArgs) {
loop {
let files = find_model_files(&args.directory);
let total_file_count = files.len();
let files = scope_to_files_to_remove(files, args.keep);
info!("Files total: {total_file_count} ; to delete: {}", files.len());
for file in files {
if !args.confirm {
info!("would have removed file (use --confirm to delete): {:?}", file);
} else {
info!("removing file: {:?}", file);
if let Err(err) = std::fs::remove_file(file) {
error!("Error removing file: {err}");
}
}
thread::sleep(Duration::from_millis(100));
}
thread::sleep(Duration::from_secs(10));
}
}
fn find_model_files(directory: &str) -> Vec<PathBuf> {
WalkDir::new(directory).into_iter()
.filter_map(Result::ok)
.filter(|e| !e.file_type().is_dir())
.map(|e| e.path().canonicalize())
.filter_map(Result::ok)
.filter(|path| {
path.file_name()
.and_then(|f| f.to_str())
.map(|f| f.starts_with(TTS_MODEL_PREFIX) || f.starts_with(VOCODER_MODEL_PREFIX))
//.map(|f| f.ends_with(".jpg") || f.starts_with("README"))
.unwrap_or(false)
})
.collect::<Vec<_>>()
}
fn scope_to_files_to_remove(files: Vec<PathBuf>, keep_count: usize) -> Vec<PathBuf> {
let mut files = file_metadata_map(files);
let mut files_to_remove = Vec::with_capacity(files.len());
while files.len() > keep_count {
if let Some((_time, file_data)) = files.pop_first() {
files_to_remove.push(file_data.path);
}
}
files_to_remove
}
fn file_metadata_map(files: Vec<PathBuf>) -> BTreeMap<SystemTime, FileData> {
files.into_iter()
.map(|path| FileData {
last_access: path.metadata()
.and_then(|m| m.accessed())
.unwrap_or(SystemTime::UNIX_EPOCH),
path,
})
.map(|file_data| (file_data.last_access, file_data))
.collect::<BTreeMap<SystemTime, FileData>>()
}
crates/lib/logging/Cargo.toml
+22
View File
@@ -0,0 +1,22 @@
[package]
name = "logging"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[lib]
# NB: Public "logging" crate is five years old and uninteresting. Stealing this name for internal use.
name = "logging"
path = "src/lib.rs"
[dependencies]
dotenv = "0.15.0"
env_logger = "0.10.0"
log = "0.4.20"
[dev-dependencies]
# None yet
crates/lib/logging/src/lib.rs
+72
View File
@@ -0,0 +1,72 @@
//! logging
//!
//! The purpose of this crate is to pin against a single version of the 'log' crate, making it
//! easier to simultaneously update across apps. We can also pack in a few useful definitions.
//!
// Never allow these
#![forbid(private_in_public)]
#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
// Okay to toggle
#![forbid(unreachable_patterns)]
#![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
// Always allow
#![allow(dead_code)]
#![allow(non_snake_case)]
use std::env;
/// Re-export.
pub use log::debug;
pub use log::error;
pub use log::info;
pub use log::log;
pub use log::trace;
pub use log::warn;
/// Name of the environment variable Rust's env logger uses
pub const ENV_RUST_LOG : &str = "RUST_LOG";
/// The default logging level
pub const DEFAULT_LOG_LEVEL: &str = "info";
/// A useful logging level
/// NB: `sqlx::query` is spammy and dumps all queries as "info"-level log lines.
/// NB: `hyper::proto::h1::io` is incredibly spammy and logs every chunk of bytes in very large files being downloaded.
pub const TYPICAL_LOG_LEVEL : &str = concat!(
"debug,",
"actix_web=info,",
"sqlx::query=warn,",
"hyper::proto::h1::io=warn,",
"storyteller_web::threads::db_health_checker_thread::db_health_checker_thread=warn,",
"http_server_common::request::get_request_ip=info," // Debug spams Rust logs
);
/// Initialize Rust's env logger.
///
/// The Rust logger reads the desired log level from the `RUST_LOG` environment variable. If this
/// isn't set, the provided default is used. If a default fallback isn't provided to this function,
/// we fall back to `"info"`.
///
/// A more robust logging config might configure on a per-component basis, eg.
/// `"tokio_reactor=warn,hyper=info,debug"`. You can read more in the `log` and `env_logger` crate
/// docs.
pub fn init_env_logger(default_if_absent: Option<&str>) {
if env::var(ENV_RUST_LOG)
.as_ref()
.ok()
.is_none()
{
let default_log_level = default_if_absent.unwrap_or(DEFAULT_LOG_LEVEL);
println!("Setting default logging level to \"{}\", override with env var {}.",
default_log_level, ENV_RUST_LOG);
env::set_var(ENV_RUST_LOG, default_log_level);
}
env_logger::init();
}