micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

make jepsen test more robust: handle errors and timeouts, fixed access key

Browse Source
This commit is contained in:
Alex Auvolat 2023-10-18 17:51:34 +02:00
parent ddd3de7fce
commit b3bf16ee27
3 changed files with 36 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
script/jepsen.garage/src/jepsen/garage.clj
View File

@ -41,7 +41,7 @@
[opts] [opts]
(let [workload ((get workloads (:workload opts)) opts) (let [workload ((get workloads (:workload opts)) opts)
garage-version (if (:increasing-timestamps opts) garage-version (if (:increasing-timestamps opts)
"03490d41d58576d7b3bcf977b2726d72a3a66ada" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
"v0.9.0")] "v0.9.0")]
(merge tests/noop-test (merge tests/noop-test
opts opts
@ -56,16 +56,23 @@
(gen/stagger (/ (:rate opts))) (gen/stagger (/ (:rate opts)))
(gen/nemesis (gen/nemesis
(cycle [(gen/sleep 5) (cycle [(gen/sleep 5)
{:type :info, :f :start} ;{:type :info, :f :partition-start}
;(gen/sleep 5)
{:type :info, :f :clock-scramble}
(gen/sleep 5) (gen/sleep 5)
{:type :info, :f :stop}])) ;{:type :info, :f :partition-stop}
;(gen/sleep 5)
{:type :info, :f :clock-scramble}]))
(gen/time-limit (:time-limit opts))) (gen/time-limit (:time-limit opts)))
(gen/log "Healing cluster") (gen/log "Healing cluster")
(gen/nemesis (gen/once {:type :info, :f :stop})) (gen/nemesis (gen/once {:type :info, :f :partition-stop}))
(gen/log "Waiting for recovery") (gen/log "Waiting for recovery")
(gen/sleep 10) (gen/sleep 10)
(gen/clients (:final-generator workload))) (gen/clients (:final-generator workload)))
:nemesis (nemesis/partition-random-halves) :nemesis (nemesis/compose
{{:partition-start :start
:partition-stop :stop} (nemesis/partition-random-halves)
{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})
:checker (checker/compose :checker (checker/compose
{:perf (checker/perf) {:perf (checker/perf)
:workload (:checker workload)}) :workload (:checker workload)})

23
script/jepsen.garage/src/jepsen/garage/daemon.clj
View File

@ -15,7 +15,8 @@
(def pidfile (str base-dir "/garage.pid")) (def pidfile (str base-dir "/garage.pid"))
(def admin-token "icanhazadmin") (def admin-token "icanhazadmin")
(def access-key "jepsen") (def access-key-id "GK8bfb6a51286071c6c9cd8bc3")
(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3")
(def bucket-name "jepsen") (def bucket-name "jepsen")
; THE GARAGE DB ; THE GARAGE DB
@ -78,10 +79,10 @@
(c/trace (c/trace
(c/exec binary :layout :apply :--version 1) (c/exec binary :layout :apply :--version 1)
(info node "garage status:" (c/exec binary :status)) (info node "garage status:" (c/exec binary :status))
(c/exec binary :key :create access-key) (c/exec binary :key :import access-key-id secret-access-key :--yes)
(c/exec binary :bucket :create bucket-name) (c/exec binary :bucket :create bucket-name)
(c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key) (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id)
(info node "key info: " (c/exec binary :key :info access-key)))) (info node "key info: " (c/exec binary :key :info access-key-id))))
(defn db (defn db
"Garage DB for a particular version" "Garage DB for a particular version"
@ -122,13 +123,9 @@
(defn creds (defn creds
"Obtain Garage credentials for node" "Obtain Garage credentials for node"
[node] [node]
(let [key-info (c/on node (c/exec binary :key :info access-key :--show-secret)) {:access-key access-key-id
[_ ak sk] (re-matches :secret-key secret-access-key
#"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" :endpoint (str "http://" node ":3900")
key-info)] :bucket bucket-name
{:access-key ak :client-config {:path-style-access-enabled true}})
:secret-key sk
:endpoint (str "http://" node ":3900")
:bucket bucket-name
:client-config {:path-style-access-enabled true}}))

22
script/jepsen.garage/src/jepsen/garage/reg.clj
View File

@ -33,19 +33,25 @@
(let [[k v] (:value op)] (let [[k v] (:value op)]
(case (:f op) (case (:f op)
:read :read
(let [value (s3/get (:creds this) k)] (try+
(assoc op :type :ok, :value (independent/tuple k value))) (let [value (s3/get (:creds this) k)]
(assoc op :type :ok, :value (independent/tuple k value)))
(catch (re-find #"Unavailable" (.getMessage %)) ex
(assoc op :type :fail, :error [:s3-error (.getMessage ex)])))
:write :write
(do (try+
(s3/put (:creds this) k v) (do
(assoc op :type :ok))))) (s3/put (:creds this) k v)
(assoc op :type :ok))
(catch (re-find #"Unavailable" (.getMessage %)) ex
(assoc op :type :fail, :error [:s3-error (.getMessage ex)]))))))
(teardown! [this test]) (teardown! [this test])
(close! [this test])) (close! [this test]))
(defn workload (defn workload
"Tests linearizable reads and writes" "Tests linearizable reads and writes"
[opts] [opts]
{:client (RegClient. nil) {:client (client/timeout 10 (RegClient. nil))
:checker (independent/checker :checker (independent/checker
(checker/compose (checker/compose
{:linear (checker/linearizable {:linear (checker/linearizable
@ -53,8 +59,8 @@
:algorithm :linear}) :algorithm :linear})
:timeline (timeline/html)})) :timeline (timeline/html)}))
:generator (independent/concurrent-generator :generator (independent/concurrent-generator
10 (/ (:concurrency opts) 10) ; divide threads in 10 groups
(range) (range) ; working on 10 keys
(fn [k] (fn [k]
(->> (->>
(gen/mix [op-get op-put op-del]) (gen/mix [op-get op-put op-del])