micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adapt tests to new syntax with public keys

Browse Source
This commit is contained in:
Alex Auvolat 2021-10-19 16:16:10 +02:00
parent 65070f3c05
commit a8ae78af0a
No known key found for this signature in database
GPG Key ID: EDABF9711E244EB1
10 changed files with 191 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
script/dev-bucket.sh
View File

@ -9,11 +9,11 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
garage bucket create eprouvette
KEY_INFO=`garage key new --name opérateur`
garage -c /tmp/config.1.toml bucket create eprouvette
KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur)
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
garage bucket allow eprouvette --read --write --key $ACCESS_KEY
garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --key $ACCESS_KEY
echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."

27
script/dev-cluster.sh
View File

@ -17,6 +17,10 @@ MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
WHICH_GARAGE=$(which garage || exit 1)
echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
NETWORK_SECRET="$(openssl rand -hex 32)"
# <<<<<<<<< BEGIN FOR LOOP ON NODES
for count in $(seq 1 3); do
CONF_PATH="/tmp/config.$count.toml"
LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
@ -26,13 +30,11 @@ block_size = 1048576 # objects are split in blocks of maximum this number of b
metadata_dir = "/tmp/garage-meta-$count"
data_dir = "/tmp/garage-data-$count"
rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node
bootstrap_peers = [
"127.0.0.1:3901",
"127.0.0.1:3902",
"127.0.0.1:3903"
]
rpc_public_addr = "127.0.0.1:$((3900+$count))"
bootstrap_peers = []
max_concurrent_rpc_requests = 12
replication_mode = "3"
rpc_secret = "$NETWORK_SECRET"
[s3_api]
api_bind_addr = "0.0.0.0:$((3910+$count))" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
@ -61,11 +63,21 @@ if [ -z "$SKIP_HTTPS" ]; then
socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
fi
(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
done
# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES
sleep 5
# Establish connections between nodes
for count in $(seq 1 3); do
NODE=$(garage -c /tmp/config.$count.toml node-id -q)
for count2 in $(seq 1 3); do
garage -c /tmp/config.$count2.toml node connect $NODE
done
done
RETRY=120
until garage status 2>&1|grep -q Healthy ; do
until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
(( RETRY-- ))
if (( RETRY <= 0 )); then
echo -en "${MAIN_LABEL} Garage did not start"
@ -74,6 +86,7 @@ until garage status 2>&1|grep -q Healthy ; do
echo -en "${MAIN_LABEL} cluster starting...\n"
sleep 1
done
echo -en "${MAIN_LABEL} cluster started\n"
wait

6
script/dev-configure.sh
View File

@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
sleep 5
RETRY=120
until garage status 2>&1|grep -q Healthy ; do
until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
(( RETRY-- ))
if (( RETRY <= 0 )); then
echo "garage did not start in time, failing."
@ -21,10 +21,10 @@ until garage status 2>&1|grep -q Healthy ; do
sleep 1
done
garage status \
garage -c /tmp/config.1.toml status \
| grep UNCONFIGURED \
| grep -Po '^[0-9a-f]+' \
| while read id; do
garage node configure -z dc1 -c 1 $id
garage -c /tmp/config.1.toml node configure -z dc1 -c 1 $id
done

6
script/test-smoke.sh
View File

@ -21,9 +21,9 @@ ${SCRIPT_FOLDER}/dev-configure.sh
${SCRIPT_FOLDER}/dev-bucket.sh
which garage
garage status
garage key list
garage bucket list
garage -c /tmp/config.1.toml status
garage -c /tmp/config.1.toml key list
garage -c /tmp/config.1.toml bucket list
dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes)
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked

35
src/garage/cli.rs
View File

@ -1,5 +1,4 @@
use std::collections::HashSet;
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use structopt::StructOpt;
@ -21,7 +20,12 @@ use crate::admin_rpc::*;
pub enum Command {
/// Run Garage server
#[structopt(name = "server")]
Server(ServerOpt),
Server,
/// Print identifier (public key) of this garage node.
/// Generates a new keypair if necessary.
#[structopt(name = "node-id")]
NodeId(NodeIdOpt),
/// Get network status
#[structopt(name = "status")]
@ -48,13 +52,6 @@ pub enum Command {
Stats(StatsOpt),
}
#[derive(StructOpt, Debug)]
pub struct ServerOpt {
/// Configuration file
#[structopt(short = "c", long = "config", default_value = "./config.toml")]
pub config_file: PathBuf,
}
#[derive(StructOpt, Debug)]
pub enum NodeOperation {
/// Connect to Garage node that is currently isolated from the system
@ -70,6 +67,13 @@ pub enum NodeOperation {
Remove(RemoveNodeOpt),
}
#[derive(StructOpt, Debug)]
pub struct NodeIdOpt {
/// Do not print usage instructions to stderr
#[structopt(short = "q", long = "quiet")]
pub(crate) quiet: bool,
}
#[derive(StructOpt, Debug)]
pub struct ConnectNodeOpt {
/// Node public key and address, in the format:
@ -384,7 +388,8 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
.any(|(id, _)| !status_keys.contains(id));
if failure_case_1 || failure_case_2 {
println!("\nFailed nodes:");
let mut failed_nodes = vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
let mut failed_nodes =
vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
for adv in status.iter().filter(|adv| !adv.is_up) {
if let Some(cfg) = config.members.get(&adv.id) {
failed_nodes.push(format!(
@ -421,14 +426,15 @@ pub async fn cmd_connect(
rpc_host: NodeID,
args: ConnectNodeOpt,
) -> Result<(), Error> {
match rpc_cli.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL).await?? {
match rpc_cli
.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL)
.await??
{
SystemRpc::Ok => {
println!("Success.");
Ok(())
}
r => {
Err(Error::BadRpc(format!("Unexpected response: {:?}", r)))
}
r => Err(Error::BadRpc(format!("Unexpected response: {:?}", r))),
}
}
@ -654,4 +660,3 @@ pub fn find_matching_node(
Ok(candidates[0])
}
}

115
src/garage/main.rs
View File

@ -9,9 +9,11 @@ mod cli;
mod repair;
mod server;
use std::path::PathBuf;
use structopt::StructOpt;
use netapp::util::parse_peer_addr;
use netapp::util::parse_and_resolve_peer_addr;
use netapp::NetworkKey;
use garage_util::error::Error;
@ -34,6 +36,10 @@ struct Opt {
#[structopt(short = "s", long = "rpc-secret")]
pub rpc_secret: Option<String>,
/// Configuration file (garage.toml)
#[structopt(short = "c", long = "config", default_value = "/etc/garage.toml")]
pub config_file: PathBuf,
#[structopt(subcommand)]
cmd: Command,
}
@ -45,16 +51,18 @@ async fn main() {
let opt = Opt::from_args();
let res = if let Command::Server(server_opt) = opt.cmd {
// Abort on panic (same behavior as in Go)
std::panic::set_hook(Box::new(|panic_info| {
error!("{}", panic_info.to_string());
std::process::abort();
}));
let res = match opt.cmd {
Command::Server => {
// Abort on panic (same behavior as in Go)
std::panic::set_hook(Box::new(|panic_info| {
error!("{}", panic_info.to_string());
std::process::abort();
}));
server::run_server(server_opt.config_file).await
} else {
cli_command(opt).await
server::run_server(opt.config_file).await
}
Command::NodeId(node_id_opt) => node_id_command(opt.config_file, node_id_opt.quiet),
_ => cli_command(opt).await,
};
if let Err(e) = res {
@ -63,16 +71,42 @@ async fn main() {
}
async fn cli_command(opt: Opt) -> Result<(), Error> {
let net_key_hex_str = &opt.rpc_secret.expect("No RPC secret provided");
let config = if opt.rpc_secret.is_none() || opt.rpc_host.is_none() {
Some(garage_util::config::read_config(opt.config_file.clone())
.map_err(|e| Error::Message(format!("Unable to read configuration file {}: {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy(), e)))?)
} else {
None
};
// Find and parse network RPC secret
let net_key_hex_str = opt
.rpc_secret
.as_ref()
.or_else(|| config.as_ref().map(|c| &c.rpc_secret))
.expect("No RPC secret provided");
let network_key = NetworkKey::from_slice(
&hex::decode(net_key_hex_str).expect("Invalid RPC secret key (bad hex)")[..],
)
.expect("Invalid RPC secret provided (wrong length)");
// Generate a temporary keypair for our RPC client
let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair();
let netapp = NetApp::new(network_key, sk);
let (id, addr) =
parse_peer_addr(&opt.rpc_host.expect("No RPC host provided")).expect("Invalid RPC host");
// Find and parse the address of the target host
let (id, addr) = if let Some(h) = opt.rpc_host {
let (id, addrs) = parse_and_resolve_peer_addr(&h).expect("Invalid RPC host");
(id, addrs[0])
} else if let Some(a) = config.as_ref().map(|c| c.rpc_public_addr).flatten() {
let node_key = garage_rpc::system::gen_node_key(&config.unwrap().metadata_dir)
.map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
(node_key.public_key(), a)
} else {
return Err(Error::Message("No RPC host provided".into()));
};
// Connect to target host
netapp.clone().try_connect(addr, id).await?;
let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
@ -80,3 +114,58 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
cli_cmd(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await
}
fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
let config = garage_util::config::read_config(config_file.clone()).map_err(|e| {
Error::Message(format!(
"Unable to read configuration file {}: {}",
config_file.to_string_lossy(),
e
))
})?;
let node_key = garage_rpc::system::gen_node_key(&config.metadata_dir)
.map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
let idstr = if let Some(addr) = config.rpc_public_addr {
let idstr = format!("{}@{}", hex::encode(&node_key.public_key()), addr);
println!("{}", idstr);
idstr
} else {
let idstr = hex::encode(&node_key.public_key());
println!("{}", idstr);
if !quiet {
eprintln!("WARNING: I don't know the public address to reach this node.");
eprintln!("In all of the instructions below, replace 127.0.0.1:3901 by the appropriate address and port.");
}
format!("{}@127.0.0.1:3901", idstr)
};
if !quiet {
eprintln!("");
eprintln!(
"To instruct a node to connect to this node, run the following command on that node:"
);
eprintln!(" garage [-c <config file path>] node connect {}", idstr);
eprintln!("");
eprintln!("Or instruct them to connect from here by running:");
eprintln!(
" garage -c {} -h <remote node> node connect {}",
config_file.to_string_lossy(),
idstr
);
eprintln!(
"where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>"
);
eprintln!("");
eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
eprintln!(" bootstrap_peers = [");
eprintln!(" \"{}\",", idstr);
eprintln!(" ...");
eprintln!(" ]");
}
Ok(())
}

2
src/rpc/rpc_helper.rs
View File

@ -14,8 +14,8 @@ pub use netapp::proto::*;
pub use netapp::{NetApp, NodeID};
use garage_util::background::BackgroundRunner;
use garage_util::error::Error;
use garage_util::data::Uuid;
use garage_util::error::Error;
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);

59
src/rpc/system.rs
View File

@ -18,8 +18,8 @@ use tokio::sync::Mutex;
use netapp::endpoint::{Endpoint, EndpointHandler};
use netapp::peering::fullmesh::FullMeshPeeringStrategy;
use netapp::proto::*;
use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
use netapp::util::parse_and_resolve_peer_addr;
use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
use garage_util::background::BackgroundRunner;
use garage_util::data::Uuid;
@ -110,7 +110,7 @@ pub struct KnownNodeInfo {
pub status: NodeStatus,
}
fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
pub fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
let mut key_file = metadata_dir.to_path_buf();
key_file.push("node_key");
if key_file.as_path().exists() {
@ -246,8 +246,12 @@ impl System {
}
async fn handle_connect(&self, node: &str) -> Result<SystemRpc, Error> {
let (pubkey, addrs) = parse_and_resolve_peer_addr(node)
.ok_or_else(|| Error::Message(format!("Unable to parse or resolve node specification: {}", node)))?;
let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| {
Error::Message(format!(
"Unable to parse or resolve node specification: {}",
node
))
})?;
let mut errors = vec![];
for ip in addrs.iter() {
match self.netapp.clone().try_connect(*ip, pubkey).await {
@ -257,7 +261,10 @@ impl System {
}
}
}
return Err(Error::Message(format!("Could not connect to specified peers. Errors: {:?}", errors)));
return Err(Error::Message(format!(
"Could not connect to specified peers. Errors: {:?}",
errors
)));
}
fn handle_pull_config(&self) -> SystemRpc {
@ -267,23 +274,25 @@ impl System {
fn handle_get_known_nodes(&self) -> SystemRpc {
let node_status = self.node_status.read().unwrap();
let known_nodes =
self.fullmesh
.get_peer_list()
.iter()
.map(|n| KnownNodeInfo {
id: n.id.into(),
addr: n.addr,
is_up: n.is_up(),
status: node_status.get(&n.id.into()).cloned().map(|(_, st)| st).unwrap_or(
NodeStatus {
hostname: "?".to_string(),
replication_factor: 0,
config_version: 0,
},
),
})
.collect::<Vec<_>>();
let known_nodes = self
.fullmesh
.get_peer_list()
.iter()
.map(|n| KnownNodeInfo {
id: n.id.into(),
addr: n.addr,
is_up: n.is_up(),
status: node_status
.get(&n.id.into())
.cloned()
.map(|(_, st)| st)
.unwrap_or(NodeStatus {
hostname: "?".to_string(),
replication_factor: 0,
config_version: 0,
}),
})
.collect::<Vec<_>>();
SystemRpc::ReturnKnownNodes(known_nodes)
}
@ -330,14 +339,14 @@ impl System {
drop(update_ring);
let self2 = self.clone();
let adv2 = adv.clone();
let adv = adv.clone();
self.background.spawn_cancellable(async move {
self2
.rpc
.broadcast(
&self2.system_endpoint,
SystemRpc::AdvertiseConfig(adv2),
RequestStrategy::with_priority(PRIO_NORMAL),
SystemRpc::AdvertiseConfig(adv),
RequestStrategy::with_priority(PRIO_HIGH),
)
.await;
Ok(())

6
src/table/replication/fullcopy.rs
View File

@ -28,11 +28,7 @@ impl TableReplication for TableFullReplication {
fn write_nodes(&self, _hash: &Hash) -> Vec<Uuid> {
let ring = self.system.ring.borrow();
ring.config
.members
.keys()
.cloned()
.collect::<Vec<_>>()
ring.config.members.keys().cloned().collect::<Vec<_>>()
}
fn write_quorum(&self) -> usize {
let nmembers = self.system.ring.borrow().config.members.len();

7
src/util/config.rs
View File

@ -6,8 +6,8 @@ use std::path::PathBuf;
use serde::de::Error as SerdeError;
use serde::{de, Deserialize};
use netapp::NodeID;
use netapp::util::parse_and_resolve_peer_addr;
use netapp::NodeID;
use crate::error::Error;
@ -117,8 +117,9 @@ where
let mut ret = vec![];
for peer in <Vec<&str>>::deserialize(deserializer)? {
let (pubkey, addrs) = parse_and_resolve_peer_addr(peer)
.ok_or_else(|| D::Error::custom(format!("Unable to parse or resolve peer: {}", peer)))?;
let (pubkey, addrs) = parse_and_resolve_peer_addr(peer).ok_or_else(|| {
D::Error::custom(format!("Unable to parse or resolve peer: {}", peer))
})?;
for ip in addrs {
ret.push((pubkey.clone(), ip));
}