micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

jepsen ssh into containers seem to work ?

Browse Source
This commit is contained in:
Alex Auvolat 2023-04-18 13:27:22 +02:00
parent 7011b71fbd
commit 17ebb65273
7 changed files with 150 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
script/jepsen.garage/destroy-tap.sh Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
USER=$(whoami)
for NODE in 1 2 3 4 5; do
sudo ip link delete microvm-n$NODE
done

64
script/jepsen.garage/flake.lock Normal file
View File

@ -0,0 +1,64 @@
{
"nodes": {
"flake-utils": {
"locked": {
"lastModified": 1678901627,
"narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"microvm": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1681747916,
"narHash": "sha256-tpWJMHWbTrFD2Nmj3Y3qYXoaTP4LFT0P0wt5zW8/aI8=",
"owner": "astro",
"repo": "microvm.nix",
"rev": "68f1b9ece0f116d5ea1d1ecaf17f7b526303df81",
"type": "github"
},
"original": {
"owner": "astro",
"repo": "microvm.nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1681737997,
"narHash": "sha256-pHhjgsIkRMu80LmVe8QoKIZB6VZGRRxFmIvsC5S89k4=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "f00994e78cd39e6fc966f0c4103f908e63284780",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"microvm": "microvm",
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

45
script/jepsen.garage/flake.nix
View File

@ -4,19 +4,56 @@
inputs.microvm.url = "github:astro/microvm.nix";
inputs.microvm.inputs.nixpkgs.follows = "nixpkgs";
outputs = { self, nixpkgs, microvm }: {
# Example nixosConfigurations entry
nixosConfigurations.my-microvm = nixpkgs.lib.nixosSystem {
outputs = { self, nixpkgs, microvm }:
with nixpkgs.lib;
let
addressMap =
{
"n1" = { ip = "10.1.0.10"; mac = "02:00:00:00:99:01"; };
"n2" = { ip = "10.2.0.10"; mac = "02:00:00:00:99:02"; };
"n3" = { ip = "10.3.0.10"; mac = "02:00:00:00:99:03"; };
"n4" = { ip = "10.4.0.10"; mac = "02:00:00:00:99:04"; };
"n5" = { ip = "10.5.0.10"; mac = "02:00:00:00:99:05"; };
};
toHostsEntry = name: { ip, ... }: "${ip} ${name}";
extraHosts =
builtins.concatStringsSep "\n"
(attrsets.mapAttrsToList toHostsEntry addressMap);
nodeConfig = hostName: { ip, mac }: nixosSystem {
system = "x86_64-linux";
modules = [
# Include the microvm module
microvm.nixosModules.microvm
# Add more modules here
{
networking.hostName = "my-microvm";
networking = {
inherit hostName extraHosts;
};
microvm.hypervisor = "cloud-hypervisor";
microvm.interfaces = [ {
inherit mac;
type = "tap";
id = "microvm-${hostName}";
} ];
services.openssh = {
enable = true;
permitRootLogin = "yes";
};
users.users.root.initialPassword = "root";
#services.garage = {
# enable = true;
# logLevel = "debug";
# settings.replication_mode = "3";
#};
}
];
};
in
{
nixosConfigurations = mapAttrs nodeConfig addressMap;
};
}

34
script/jepsen.garage/garage-cluster.nix
View File

@ -29,30 +29,36 @@ let
services.openssh = {
enable = true;
permitRootLogin = "yes";
kexAlgorithms = [ "curve25519-sha256@libssh.org" "ecdh-sha2-nistp256" "ecdh-sha2-nistp384" "ecdh-sha2-nistp521" "diffie-hellman-group-exchange-sha256" "diffie-hellman-group14-sha1" "diffie-hellman-group-exchange-sha1" "diffie-hellman-group1-sha1" ];
};
users.users.root.initialPassword = "root";
users.users.root.openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9"
];
system.stateVersion = "22.11";
services.garage = {
enable = true;
logLevel = "debug";
settings.replication_mode = "3";
settings = {
replication_mode = "3";
db_engine = "lmdb";
rpc_secret = "b597bb28ebdc90cdc4f15712733ca678cfb9a7e0311e0b9e93db9610fc3685e6";
rpc_bind_addr = "0.0.0.0:3901";
s3_api = {
region = "garage";
api_bind_addr = "0.0.0.0:3900";
};
k2v_api.api_bind_addr = "0.0.0.0:3902";
admin = {
api_bind_addr = "0.0.0.0:3903";
admin_token = "icanhazadmin";
};
};
};
# Workaround for nixos-container issue
# (see https://github.com/NixOS/nixpkgs/issues/67265 and
# https://github.com/NixOS/nixpkgs/pull/81371#issuecomment-605526099).
# The etcd service is of type "notify", which means that
# etcd would not be considered started until etcd is fully online;
# however, since NixOS container networking only works sometime *after*
# multi-user.target, we forgo etcd's notification entirely.
systemd.services.etcd.serviceConfig.Type = lib.mkForce "exec";
systemd.services.etcd.serviceConfig.StandardOutput = "file:/var/log/etcd.log";
systemd.services.etcd.serviceConfig.StandardError = "file:/var/log/etcd.log";
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
networking.firewall.allowedTCPPorts = [ 3901 3900 3902 3903 ];
};
};
in

5
script/jepsen.garage/nodes Normal file
View File

@ -0,0 +1,5 @@
n1.containers
n2.containers
n3.containers
n4.containers
n5.containers

2
script/jepsen.garage/project.clj
View File

@ -5,5 +5,5 @@
:url "https://www.gnu.org/licenses/gpl-3.0.en.html"}
:main jepsen.garage
:dependencies [[org.clojure/clojure "1.11.1"]
[jepsen "0.2.1-SNAPSHOT"]]
[jepsen "0.3.2-SNAPSHOT"]]
:repl-options {:init-ns jepsen.garage})

10
script/jepsen.garage/setup-tap.sh Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
USER=$(whoami)
for NODE in 1 2 3 4 5; do
sudo ip tuntap add microvm-n$NODE mode tap user $USER
sudo ip addr add dev microvm-n$NODE 10.$NODE.0.1
done