gsadmin/garage
1
0
Fork 0
mirror of https://github.com/deuxfleurs-org/garage.git synced 2026-08-30 08:49:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

jepsen: fix the list-objects call (?)

Browse Source
This commit is contained in:
Alex Auvolat
2023-10-19 23:40:55 +02:00
parent da8b170748
commit ef662822c9
3 changed files with 52 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
script/jepsen.garage/src/jepsen/garage/s3api.clj
+14 -13
View File
@@ -31,19 +31,20 @@
:input-stream bytes-stream
:metadata {:content-length (count some-bytes)}))))
(defn list-inner [creds prefix ct accum]
(let [list-result (s3/list-objects-v2 creds
{:bucket-name (:bucket creds)
:prefix prefix
:continuation-token ct})
new-object-summaries (:object-summaries list-result)
new-objects (map (fn [d] (:key d)) new-object-summaries)
objects (concat new-objects accum)]
(info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects)
(if (:truncated? list-result)
(list-inner creds prefix (:next-continuation-token list-result) objects)
objects)))
(defn list
"Helper for ListObjects -- just lists everything in the bucket"
[creds prefix]
(defn list-inner [ct accum]
(let [list-result (s3/list-objects-v2 creds
{:bucket-name (:bucket creds)
:prefix prefix
:continuation-token ct})
new-object-summaries (:object-summaries list-result)
new-objects (map (fn [d] (:key d)) new-object-summaries)
objects (concat new-objects accum)]
(info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects)
(if (:truncated? list-result)
(list-inner (:next-continuation-token list-result) objects)
objects)))
(list-inner nil []))
(info "in s3/list creds:" creds ", prefix:" prefix)
(list-inner creds prefix nil []))
script/jepsen.garage/src/jepsen/garage/set.clj
+9 -7
View File
@@ -44,13 +44,15 @@
(util/timeout
10000
(assoc op :type :fail, :error ::timeout)
(let [items (s3/list (:creds this) prefix)]
(info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")")
(let [items-stripped (map (fn [o]
(assert (str/starts-with? o prefix))
(str/replace-first o prefix "")) items)
items-set (set (map parse-long items-stripped))]
(assoc op :type :ok, :value (independent/tuple k items-set))))))))
(do
(info "call s3/list creds: " (:creds this) ", prefix:" prefix)
(let [items (s3/list (:creds this) prefix)]
(info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")")
(let [items-stripped (map (fn [o]
(assert (str/starts-with? o prefix))
(str/replace-first o prefix "")) items)
items-set (set (map parse-long items-stripped))]
(assoc op :type :ok, :value (independent/tuple k items-set)))))))))
(teardown! [this test])
(close! [this test]))