micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Prepare for v0.3.0 and add migration path from v0.2.1.x

Browse Source
This commit is contained in:
Alex Auvolat 2021-05-28 13:58:47 +02:00
parent ddb2b29bfd
commit b9127dd6f8
No known key found for this signature in database
GPG Key ID: EDABF9711E244EB1
14 changed files with 161 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
Cargo.lock generated
View File

@ -350,16 +350,16 @@ dependencies = [
[[package]] [[package]]
name = "garage" name = "garage"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"bytes", "bytes",
"futures", "futures",
"futures-util", "futures-util",
"garage_api", "garage_api",
"garage_model", "garage_model",
"garage_rpc", "garage_rpc 0.3.0",
"garage_table", "garage_table",
"garage_util", "garage_util 0.3.0",
"garage_web", "garage_web",
"git-version", "git-version",
"hex", "hex",
@ -376,7 +376,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api" name = "garage_api"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"base64", "base64",
"bytes", "bytes",
@ -387,7 +387,7 @@ dependencies = [
"futures-util", "futures-util",
"garage_model", "garage_model",
"garage_table", "garage_table",
"garage_util", "garage_util 0.3.0",
"hex", "hex",
"hmac", "hmac",
"http", "http",
@ -407,14 +407,14 @@ dependencies = [
[[package]] [[package]]
name = "garage_model" name = "garage_model"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"futures", "futures",
"futures-util", "futures-util",
"garage_rpc", "garage_rpc 0.3.0",
"garage_table", "garage_table",
"garage_util", "garage_util 0.3.0",
"hex", "hex",
"log", "log",
"rand", "rand",
@ -428,12 +428,40 @@ dependencies = [
[[package]] [[package]]
name = "garage_rpc" name = "garage_rpc"
version = "0.2.1" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c182633cebe4abed9594afb14770fc45402513765d38a4b19659ae0ccb2a2f"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"bytes", "bytes",
"futures", "futures",
"futures-util", "futures-util",
"garage_util", "garage_util 0.2.1",
"gethostname",
"hex",
"http",
"hyper",
"hyper-rustls",
"log",
"rmp-serde",
"rustls",
"serde",
"serde_json",
"tokio",
"tokio-rustls",
"tokio-stream",
"webpki",
]
[[package]]
name = "garage_rpc"
version = "0.3.0"
dependencies = [
"arc-swap",
"bytes",
"futures",
"futures-util",
"garage_rpc 0.2.1",
"garage_util 0.3.0",
"gethostname", "gethostname",
"hex", "hex",
"http", "http",
@ -452,13 +480,13 @@ dependencies = [
[[package]] [[package]]
name = "garage_table" name = "garage_table"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"bytes", "bytes",
"futures", "futures",
"futures-util", "futures-util",
"garage_rpc", "garage_rpc 0.3.0",
"garage_util", "garage_util 0.3.0",
"hexdump", "hexdump",
"log", "log",
"rand", "rand",
@ -472,6 +500,33 @@ dependencies = [
[[package]] [[package]]
name = "garage_util" name = "garage_util"
version = "0.2.1" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aef76d3779e406a16fdcaffe8d86b8ae2943a549d2b33f2c20930838764464c0"
dependencies = [
"blake2",
"chrono",
"err-derive",
"futures",
"hex",
"http",
"hyper",
"log",
"rand",
"rmp-serde",
"rustls",
"serde",
"serde_json",
"sha2",
"sled",
"tokio",
"toml",
"webpki",
"xxhash-rust",
]
[[package]]
name = "garage_util"
version = "0.3.0"
dependencies = [ dependencies = [
"blake2", "blake2",
"chrono", "chrono",
@ -496,14 +551,14 @@ dependencies = [
[[package]] [[package]]
name = "garage_web" name = "garage_web"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"err-derive", "err-derive",
"futures", "futures",
"garage_api", "garage_api",
"garage_model", "garage_model",
"garage_table", "garage_table",
"garage_util", "garage_util 0.3.0",
"http", "http",
"hyper", "hyper",
"idna", "idna",

17
Makefile
View File

@ -1,20 +1,5 @@
BIN=target/release/garage
DOCKER=lxpz/garage_amd64
all: all:
clear; cargo build clear; cargo build
$(BIN): release:
RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features
$(BIN).stripped: $(BIN)
cp $^ $@
strip $@
docker: $(BIN).stripped
docker pull archlinux:latest
docker build -t $(DOCKER):$(TAG) .
docker push $(DOCKER):$(TAG)
docker tag $(DOCKER):$(TAG) $(DOCKER):latest
docker push $(DOCKER):latest

22
config.dev.toml
View File

@ -1,22 +0,0 @@
block_size = 1048576 # objects are split in blocks of maximum this number of bytes
metadata_dir = "/tmp/garage-meta"
data_dir = "/tmp/garage-data"
rpc_bind_addr = "[::]:3901" # the port other Garage nodes will use to talk to this node
bootstrap_peers = []
max_concurrent_rpc_requests = 12
data_replication_factor = 3
meta_replication_factor = 3
meta_epidemic_fanout = 3
[s3_api]
api_bind_addr = "[::1]:3900" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
s3_region = "garage" # set this to anything. S3 API calls will fail if they are not made against the region set here.
[s3_web]
bind_addr = "[::1]:3902"
root_domain = ".garage.tld"
index = "index.html"

20
make_docker.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/sh
BIN=target/release/garage
DOCKER=lxpz/garage_amd64
TAG=$1
if [ -z "$1" ]; then
echo "Usage: $0 <tag>"
exit 1
fi
RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features
cp $BIN $BIN.stripped
strip $BIN.stripped
docker pull archlinux:latest
docker build -t $DOCKER:$TAG .
docker push $DOCKER:$TAG
docker tag $DOCKER:$TAG $DOCKER:latest
docker push $DOCKER:latest

8
src/api/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "garage_api" name = "garage_api"
version = "0.2.1" version = "0.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -13,9 +13,9 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_model = { version = "0.2.1", path = "../model" } garage_model = { version = "0.3.0", path = "../model" }
garage_table = { version = "0.2.1", path = "../table" } garage_table = { version = "0.3.0", path = "../table" }
garage_util = { version = "0.2.1", path = "../util" } garage_util = { version = "0.3.0", path = "../util" }
base64 = "0.13" base64 = "0.13"
bytes = "1.0" bytes = "1.0"

14
src/garage/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "garage" name = "garage"
version = "0.2.1" version = "0.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,12 +14,12 @@ path = "main.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_api = { version = "0.2.1", path = "../api" } garage_api = { version = "0.3.0", path = "../api" }
garage_model = { version = "0.2.1", path = "../model" } garage_model = { version = "0.3.0", path = "../model" }
garage_rpc = { version = "0.2.1", path = "../rpc" } garage_rpc = { version = "0.3.0", path = "../rpc" }
garage_table = { version = "0.2.1", path = "../table" } garage_table = { version = "0.3.0", path = "../table" }
garage_util = { version = "0.2.1", path = "../util" } garage_util = { version = "0.3.0", path = "../util" }
garage_web = { version = "0.2.1", path = "../web" } garage_web = { version = "0.3.0", path = "../web" }
bytes = "1.0" bytes = "1.0"
git-version = "0.3.4" git-version = "0.3.4"

8
src/garage/cli.rs
View File

@ -467,9 +467,7 @@ pub async fn cmd_configure(
"Please specify a capacity with the -c flag, or set node explicitly as gateway with -g".into())), "Please specify a capacity with the -c flag, or set node explicitly as gateway with -g".into())),
}; };
NetworkConfigEntry { NetworkConfigEntry {
zone: args zone: args.zone.expect("Please specifiy a zone with the -z flag"),
.zone
.expect("Please specifiy a zone with the -z flag"),
capacity, capacity,
tag: args.tag.unwrap_or_default(), tag: args.tag.unwrap_or_default(),
} }
@ -481,9 +479,7 @@ pub async fn cmd_configure(
_ => old.capacity, _ => old.capacity,
}; };
NetworkConfigEntry { NetworkConfigEntry {
zone: args zone: args.zone.unwrap_or_else(|| old.zone.to_string()),
.zone
.unwrap_or_else(|| old.zone.to_string()),
capacity, capacity,
tag: args.tag.unwrap_or_else(|| old.tag.to_string()), tag: args.tag.unwrap_or_else(|| old.tag.to_string()),
} }

8
src/model/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "garage_model" name = "garage_model"
version = "0.2.1" version = "0.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -13,9 +13,9 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_rpc = { version = "0.2.1", path = "../rpc" } garage_rpc = { version = "0.3.0", path = "../rpc" }
garage_table = { version = "0.2.1", path = "../table" } garage_table = { version = "0.3.0", path = "../table" }
garage_util = { version = "0.2.1", path = "../util" } garage_util = { version = "0.3.0", path = "../util" }
arc-swap = "1.0" arc-swap = "1.0"
hex = "0.4" hex = "0.4"

6
src/rpc/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "garage_rpc" name = "garage_rpc"
version = "0.2.1" version = "0.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -13,7 +13,9 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_util = { version = "0.2.1", path = "../util" } garage_util = { version = "0.3.0", path = "../util" }
garage_rpc_021 = { package = "garage_rpc", version = "0.2.1" }
arc-swap = "1.0" arc-swap = "1.0"
bytes = "1.0" bytes = "1.0"

20
src/rpc/membership.rs
View File

@ -240,11 +240,21 @@ impl System {
let net_config = match persist_config.load() { let net_config = match persist_config.load() {
Ok(x) => x, Ok(x) => x,
Err(e) => { Err(e) => {
info!( match Persister::<garage_rpc_021::ring::NetworkConfig>::new(
"No valid previous network configuration stored ({}), starting fresh.", &metadata_dir,
e "network_config",
); )
NetworkConfig::new() .load()
{
Ok(old_config) => NetworkConfig::migrate_from_021(old_config),
Err(e2) => {
info!(
"No valid previous network configuration stored ({}, {}), starting fresh.",
e, e2
);
NetworkConfig::new()
}
}
} }
}; };

28
src/rpc/ring.rs
View File

@ -38,6 +38,31 @@ impl NetworkConfig {
version: 0, version: 0,
} }
} }
pub(crate) fn migrate_from_021(old: garage_rpc_021::ring::NetworkConfig) -> Self {
let members = old
.members
.into_iter()
.map(|(id, conf)| {
(
Hash::try_from(id.as_slice()).unwrap(),
NetworkConfigEntry {
zone: conf.datacenter,
capacity: if conf.capacity == 0 {
None
} else {
Some(conf.capacity)
},
tag: conf.tag,
},
)
})
.collect();
Self {
members,
version: old.version,
}
}
} }
/// The overall configuration of one (possibly remote) node /// The overall configuration of one (possibly remote) node
@ -178,8 +203,7 @@ impl Ring {
.iter() .iter()
.map(|(_id, info)| info.zone.as_str()) .map(|(_id, info)| info.zone.as_str())
.collect::<HashSet<&str>>(); .collect::<HashSet<&str>>();
if (p_zns.len() < n_zones if (p_zns.len() < n_zones && !p_zns.contains(&node_info.zone.as_str()))
&& !p_zns.contains(&node_info.zone.as_str()))
|| (p_zns.len() == n_zones || (p_zns.len() == n_zones
&& !partitions[qv].iter().any(|(id, _i)| id == node_id)) && !partitions[qv].iter().any(|(id, _i)| id == node_id))
{ {

6
src/table/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "garage_table" name = "garage_table"
version = "0.2.1" version = "0.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -13,8 +13,8 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_rpc = { version = "0.2.1", path = "../rpc" } garage_rpc = { version = "0.3.0", path = "../rpc" }
garage_util = { version = "0.2.1", path = "../util" } garage_util = { version = "0.3.0", path = "../util" }
bytes = "1.0" bytes = "1.0"
hexdump = "0.1" hexdump = "0.1"

2
src/util/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "garage_util" name = "garage_util"
version = "0.2.1" version = "0.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"

10
src/web/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "garage_web" name = "garage_web"
version = "0.2.1" version = "0.3.0"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"] authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -13,10 +13,10 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_api = { version = "0.2.1", path = "../api" } garage_api = { version = "0.3.0", path = "../api" }
garage_model = { version = "0.2.1", path = "../model" } garage_model = { version = "0.3.0", path = "../model" }
garage_util = { version = "0.2.1", path = "../util" } garage_util = { version = "0.3.0", path = "../util" }
garage_table = { version = "0.2.1", path = "../table" } garage_table = { version = "0.3.0", path = "../table" }
err-derive = "0.3" err-derive = "0.3"
idna = "0.2" idna = "0.2"