jepsen: s3 gets and puts
This commit is contained in:
Alex Auvolat
parent
ca4cc7e44f
commit
bc11701999
6
script/jepsen.garage/Vagrantfile
vendored
6
script/jepsen.garage/Vagrantfile
vendored
@ -8,7 +8,7 @@ def vm(config, hostname, ip)
|
||||
end
|
||||
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vm.box = "generic/alpine38"
|
||||
config.vm.box = "generic/debian10"
|
||||
|
||||
config.vm.provider "virtualbox" do |vb|
|
||||
vb.gui = false
|
||||
@ -17,8 +17,8 @@ Vagrant.configure("2") do |config|
|
||||
|
||||
config.vm.provision "shell", inline: <<-SHELL
|
||||
echo "root:root" | chpasswd
|
||||
apk update
|
||||
apk add wget
|
||||
mkdir -p /root/.ssh
|
||||
echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" >> /root/.ssh/authorized_keys
|
||||
SHELL
|
||||
|
||||
config.vm.define "n1" do |config| vm(config, "n1", "192.168.56.21") end
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
USER=$(whoami)
|
||||
|
||||
for NODE in 1 2 3 4 5; do
|
||||
sudo ip link delete microvm-n$NODE
|
||||
done
|
||||
|
||||
|
||||
@ -1,70 +0,0 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
let
|
||||
unstable = import ./unstable.nix;
|
||||
addressMap =
|
||||
{
|
||||
"n1" = { localAddress = "10.233.0.101"; hostAddress = "10.233.1.101"; };
|
||||
"n2" = { localAddress = "10.233.0.102"; hostAddress = "10.233.1.102"; };
|
||||
"n3" = { localAddress = "10.233.0.103"; hostAddress = "10.233.1.103"; };
|
||||
"n4" = { localAddress = "10.233.0.104"; hostAddress = "10.233.1.104"; };
|
||||
"n5" = { localAddress = "10.233.0.105"; hostAddress = "10.233.1.105"; };
|
||||
};
|
||||
toHostsEntry = name: { localAddress, ... }: "${localAddress} ${name}";
|
||||
extraHosts =
|
||||
builtins.concatStringsSep "\n"
|
||||
(lib.attrsets.mapAttrsToList toHostsEntry addressMap);
|
||||
nodeConfig = hostName: { localAddress, hostAddress }: {
|
||||
inherit localAddress hostAddress;
|
||||
|
||||
ephemeral = true;
|
||||
autoStart = true;
|
||||
privateNetwork = true;
|
||||
|
||||
config = { config, pkgs, ... }:
|
||||
{
|
||||
networking = {
|
||||
inherit hostName extraHosts;
|
||||
};
|
||||
|
||||
services.openssh = {
|
||||
enable = true;
|
||||
permitRootLogin = "yes";
|
||||
kexAlgorithms = [ "curve25519-sha256@libssh.org" "ecdh-sha2-nistp256" "ecdh-sha2-nistp384" "ecdh-sha2-nistp521" "diffie-hellman-group-exchange-sha256" "diffie-hellman-group14-sha1" "diffie-hellman-group-exchange-sha1" "diffie-hellman-group1-sha1" ];
|
||||
};
|
||||
users.users.root.initialPassword = "root";
|
||||
users.users.root.openssh.authorizedKeys.keys = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9"
|
||||
];
|
||||
|
||||
system.stateVersion = "22.11";
|
||||
|
||||
services.garage = {
|
||||
enable = true;
|
||||
logLevel = "debug";
|
||||
settings = {
|
||||
replication_mode = "3";
|
||||
db_engine = "lmdb";
|
||||
rpc_secret = "b597bb28ebdc90cdc4f15712733ca678cfb9a7e0311e0b9e93db9610fc3685e6";
|
||||
rpc_bind_addr = "0.0.0.0:3901";
|
||||
s3_api = {
|
||||
region = "garage";
|
||||
api_bind_addr = "0.0.0.0:3900";
|
||||
};
|
||||
k2v_api.api_bind_addr = "0.0.0.0:3902";
|
||||
admin = {
|
||||
api_bind_addr = "0.0.0.0:3903";
|
||||
admin_token = "icanhazadmin";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 3901 3900 3902 3903 ];
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
containers = lib.attrsets.mapAttrs nodeConfig addressMap;
|
||||
networking = {
|
||||
inherit extraHosts;
|
||||
};
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
n1.containers
|
||||
n2.containers
|
||||
n3.containers
|
||||
n4.containers
|
||||
n5.containers
|
||||
@ -5,5 +5,6 @@
|
||||
:url "https://www.gnu.org/licenses/gpl-3.0.en.html"}
|
||||
:main jepsen.garage
|
||||
:dependencies [[org.clojure/clojure "1.11.1"]
|
||||
[jepsen "0.3.2-SNAPSHOT"]]
|
||||
[jepsen "0.3.2-SNAPSHOT"]
|
||||
[amazonica "0.3.163"]]
|
||||
:repl-options {:init-ns jepsen.garage})
|
||||
|
||||
@ -1,10 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
USER=$(whoami)
|
||||
|
||||
for NODE in 1 2 3 4 5; do
|
||||
sudo ip tuntap add microvm-n$NODE mode tap user $USER
|
||||
sudo ip addr add dev microvm-n$NODE 10.$NODE.0.1
|
||||
done
|
||||
|
||||
|
||||
@ -1,14 +1,146 @@
|
||||
(ns jepsen.garage
|
||||
(:require [jepsen.cli :as cli]
|
||||
[jepsen.tests :as tests]))
|
||||
(:require [clojure.tools.logging :refer :all]
|
||||
[clojure.string :as str]
|
||||
[jepsen [cli :as cli]
|
||||
[client :as client]
|
||||
[control :as c]
|
||||
[db :as db]
|
||||
[generator :as gen]
|
||||
[tests :as tests]]
|
||||
[jepsen.control.util :as cu]
|
||||
[jepsen.os.debian :as debian]
|
||||
[amazonica.aws.s3 :as s3]
|
||||
[amazonica.aws.s3transfer :as s3transfer]))
|
||||
|
||||
(def dir "/opt/garage")
|
||||
(def binary (str dir "/garage"))
|
||||
(def logfile (str dir "/garage.log"))
|
||||
(def pidfile (str dir "/garage.pid"))
|
||||
|
||||
(def grg-admin-token "icanhazadmin")
|
||||
(def grg-key "jepsen")
|
||||
(def grg-bucket "jepsen")
|
||||
(def grg-object "1")
|
||||
|
||||
(defn db
|
||||
"Garage DB for a particular version"
|
||||
[version]
|
||||
(reify db/DB
|
||||
(setup! [_ test node]
|
||||
(info node "installing garage" version)
|
||||
(c/su
|
||||
(c/exec :mkdir :-p dir)
|
||||
(let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
|
||||
cache (cu/wget! url)]
|
||||
(c/exec :cp cache binary))
|
||||
(c/exec :chmod :+x binary)
|
||||
(cu/write-file!
|
||||
(str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
|
||||
"rpc_bind_addr = \"0.0.0.0:3901\"\n"
|
||||
"rpc_public_addr = \"" node ":3901\"\n"
|
||||
"db_engine = \"lmdb\"\n"
|
||||
"replication_mode = \"3\"\n"
|
||||
"data_dir = \"" dir "/data\"\n"
|
||||
"metadata_dir = \"" dir "/meta\"\n"
|
||||
"[s3_api]\n"
|
||||
"s3_region = \"us-east-1\"\n"
|
||||
"api_bind_addr = \"0.0.0.0:3900\"\n"
|
||||
"[k2v_api]\n"
|
||||
"api_bind_addr = \"0.0.0.0:3902\"\n"
|
||||
"[admin]\n"
|
||||
"api_bind_addr = \"0.0.0.0:3903\"\n"
|
||||
"admin_token = \"" grg-admin-token "\"\n")
|
||||
"/etc/garage.toml")
|
||||
(cu/start-daemon!
|
||||
{:logfile logfile
|
||||
:pidfile pidfile
|
||||
:chdir dir}
|
||||
binary
|
||||
:server)
|
||||
(Thread/sleep 100)
|
||||
(let [node-id (c/exec binary :node :id :-q)]
|
||||
(info node "node id:" node-id)
|
||||
(c/on-many (:nodes test)
|
||||
(c/exec binary :node :connect node-id))
|
||||
(c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node))
|
||||
(if (= node (first (:nodes test)))
|
||||
(do
|
||||
(Thread/sleep 2000)
|
||||
(c/exec binary :layout :apply :--version 1)
|
||||
(info node "garage status:" (c/exec binary :status))
|
||||
(c/exec binary :key :new :--name grg-key)
|
||||
(c/exec binary :bucket :create grg-bucket)
|
||||
(c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
|
||||
(info node "key info: " (c/exec binary :key :info grg-key))))))
|
||||
(teardown! [_ test node]
|
||||
(info node "tearing down garage" version)
|
||||
(c/su
|
||||
(cu/stop-daemon! binary pidfile)
|
||||
(c/exec :rm :-rf dir)))
|
||||
db/LogFiles
|
||||
(log-files [_ test node]
|
||||
[logfile])))
|
||||
|
||||
(defn op-get [_ _] {:type :invoke, :f :get-object, :value nil})
|
||||
(defn op-put [_ _] {:type :invoke, :f :put-object, :value (str (rand-int 50))})
|
||||
(defn op-del [_ _] {:type :invoke, :f :del-object, :value nil})
|
||||
|
||||
(defrecord Client [creds]
|
||||
client/Client
|
||||
(open! [this test node]
|
||||
(let [key-info (c/on node (c/exec binary :key :info grg-key))
|
||||
[_ ak sk] (re-matches
|
||||
#"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
|
||||
key-info)
|
||||
creds {:access-key ak
|
||||
:secret-key sk
|
||||
:endpoint (str "http://" node ":3900")
|
||||
:client-config {:path-style-access-enabled true}}]
|
||||
(info node "s3 credentials:" creds)
|
||||
(assoc this :creds creds)))
|
||||
(setup! [this test])
|
||||
(invoke! [this test op]
|
||||
(case (:f op)
|
||||
:get-object
|
||||
(let [value-bytes (try
|
||||
(-> (s3/get-object (:creds this) grg-bucket grg-object)
|
||||
:input-stream
|
||||
slurp)
|
||||
(catch Exception e nil))]
|
||||
(assoc op :type :ok, :value value-bytes))
|
||||
:put-object
|
||||
(let [some-bytes (.getBytes (:value op) "UTF-8")
|
||||
bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
|
||||
(s3/put-object (:creds this)
|
||||
:bucket-name grg-bucket
|
||||
:key grg-object
|
||||
:input-stream bytes-stream
|
||||
:metadata {:content-length (count some-bytes)})
|
||||
(assoc op :type :ok))
|
||||
:del-object
|
||||
(do
|
||||
(s3/delete-object (:creds this)
|
||||
:bucket-name grg-bucket
|
||||
:key grg-object)
|
||||
(assoc op :type :ok, :value nil))))
|
||||
(teardown! [this test])
|
||||
(close! [this test]))
|
||||
|
||||
(defn garage-test
|
||||
"Given an options map from the command line runner (e.g. :nodes, :ssh,
|
||||
:concurrency, ...), constructs a test map."
|
||||
[opts]
|
||||
(merge tests/noop-test
|
||||
{:pure-generators true}
|
||||
opts))
|
||||
opts
|
||||
{:pure-generators true
|
||||
:name "garage"
|
||||
:os debian/os
|
||||
:db (db "v0.8.2")
|
||||
:client (Client. nil)
|
||||
:generator (->> (gen/mix [op-get op-put])
|
||||
(gen/stagger 1)
|
||||
(gen/nemesis nil)
|
||||
(gen/time-limit 15))}))
|
||||
|
||||
(defn -main
|
||||
"Handles command line arguments. Can either run a test, or a web server for
|
||||
|
||||
Loading…
Reference in New Issue
Block a user