micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New buckets for 0.6.0: make bucket id a SK and not a HK, CLI updates

Browse Source
This commit is contained in:
Alex Auvolat 2022-01-03 19:06:04 +01:00
parent 1bcd6fabbd
commit df35feba18
No known key found for this signature in database
GPG Key ID: EDABF9711E244EB1
9 changed files with 86 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/api/s3_bucket.rs
View File

@ -58,21 +58,17 @@ pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result<Respo
let mut aliases = HashMap::new(); let mut aliases = HashMap::new();
for bucket_id in ids.iter() { for bucket_id in ids.iter() {
let bucket = garage.bucket_table.get(bucket_id, &EmptyKey).await?; let bucket = garage.bucket_table.get(&EmptyKey, bucket_id).await?;
if let Some(bucket) = bucket { if let Some(bucket) = bucket {
if let Deletable::Present(param) = bucket.state { for (alias, _, _active) in bucket.aliases().iter().filter(|(_, _, active)| *active) {
for (alias, _, active) in param.aliases.items() { let alias_opt = garage.bucket_alias_table.get(&EmptyKey, alias).await?;
if *active { if let Some(alias_ent) = alias_opt {
let alias_ent = garage.bucket_alias_table.get(&EmptyKey, alias).await?; if *alias_ent.state.get() == Some(*bucket_id) {
if let Some(alias_ent) = alias_ent {
if let Some(alias_bucket) = alias_ent.state.get() {
if alias_bucket == bucket_id {
aliases.insert(alias_ent.name().to_string(), *bucket_id); aliases.insert(alias_ent.name().to_string(), *bucket_id);
} }
} }
} }
} if let Deletable::Present(param) = bucket.state {
}
buckets_by_id.insert(bucket_id, param); buckets_by_id.insert(bucket_id, param);
} }
} }

4
src/api/s3_website.rs
View File

@ -20,7 +20,7 @@ pub async fn handle_delete_website(
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let mut bucket = garage let mut bucket = garage
.bucket_table .bucket_table
.get(&bucket_id, &EmptyKey) .get(&EmptyKey, &bucket_id)
.await? .await?
.ok_or(Error::NotFound)?; .ok_or(Error::NotFound)?;
@ -48,7 +48,7 @@ pub async fn handle_put_website(
let mut bucket = garage let mut bucket = garage
.bucket_table .bucket_table
.get(&bucket_id, &EmptyKey) .get(&EmptyKey, &bucket_id)
.await? .await?
.ok_or(Error::NotFound)?; .ok_or(Error::NotFound)?;

10
src/garage/admin.rs
View File

@ -38,7 +38,7 @@ pub enum AdminRpc {
// Replies // Replies
Ok(String), Ok(String),
BucketList(Vec<BucketAlias>), BucketList(Vec<Bucket>),
BucketInfo(Bucket, HashMap<String, Key>), BucketInfo(Bucket, HashMap<String, Key>),
KeyList(Vec<(String, String)>), KeyList(Vec<(String, String)>),
KeyInfo(Key, HashMap<Uuid, Bucket>), KeyInfo(Key, HashMap<Uuid, Bucket>),
@ -76,12 +76,12 @@ impl AdminRpcHandler {
} }
async fn handle_list_buckets(&self) -> Result<AdminRpc, Error> { async fn handle_list_buckets(&self) -> Result<AdminRpc, Error> {
let bucket_aliases = self let buckets = self
.garage .garage
.bucket_alias_table .bucket_table
.get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000) .get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000)
.await?; .await?;
Ok(AdminRpc::BucketList(bucket_aliases)) Ok(AdminRpc::BucketList(buckets))
} }
async fn handle_bucket_info(&self, query: &BucketOpt) -> Result<AdminRpc, Error> { async fn handle_bucket_info(&self, query: &BucketOpt) -> Result<AdminRpc, Error> {
@ -536,7 +536,7 @@ impl AdminRpcHandler {
.items() .items()
.iter() .iter()
{ {
if let Some(b) = self.garage.bucket_table.get(id, &EmptyKey).await? { if let Some(b) = self.garage.bucket_table.get(&EmptyKey, id).await? {
relevant_buckets.insert(*id, b); relevant_buckets.insert(*id, b);
} }
} }

15
src/garage/cli/cmd.rs
View File

@ -165,24 +165,13 @@ pub async fn cmd_admin(
println!("{}", msg); println!("{}", msg);
} }
AdminRpc::BucketList(bl) => { AdminRpc::BucketList(bl) => {
println!("List of buckets:"); print_bucket_list(bl);
let mut table = vec![];
for alias in bl {
if let Some(alias_bucket) = alias.state.get() {
table.push(format!("\t{}\t{:?}", alias.name(), alias_bucket));
}
}
format_table(table);
println!("Buckets that don't have a global alias (i.e. that only exist in the namespace of an access key) are not shown.");
} }
AdminRpc::BucketInfo(bucket, rk) => { AdminRpc::BucketInfo(bucket, rk) => {
print_bucket_info(&bucket, &rk); print_bucket_info(&bucket, &rk);
} }
AdminRpc::KeyList(kl) => { AdminRpc::KeyList(kl) => {
println!("List of keys:"); print_key_list(kl);
for key in kl {
println!("{}\t{}", key.0, key.1);
}
} }
AdminRpc::KeyInfo(key, rb) => { AdminRpc::KeyInfo(key, rb) => {
print_key_info(&key, &rb); print_key_info(&key, &rb);

44
src/garage/cli/util.rs
View File

@ -7,6 +7,46 @@ use garage_util::error::*;
use garage_model::bucket_table::*; use garage_model::bucket_table::*;
use garage_model::key_table::*; use garage_model::key_table::*;
pub fn print_bucket_list(bl: Vec<Bucket>) {
println!("List of buckets:");
let mut table = vec![];
for bucket in bl {
let aliases = bucket
.aliases()
.iter()
.filter(|(_, _, active)| *active)
.map(|(name, _, _)| name.to_string())
.collect::<Vec<_>>();
let local_aliases_n = match bucket
.local_aliases()
.iter()
.filter(|(_, _, active)| *active)
.count()
{
0 => "".into(),
1 => "1 local alias".into(),
n => format!("{} local aliases", n),
};
table.push(format!(
"\t{}\t{}\t{}",
aliases.join(","),
local_aliases_n,
hex::encode(bucket.id)
));
}
format_table(table);
}
pub fn print_key_list(kl: Vec<(String, String)>) {
println!("List of keys:");
let mut table = vec![];
for key in kl {
table.push(format!("\t{}\t{}", key.0, key.1));
}
format_table(table);
}
pub fn print_key_info(key: &Key, relevant_buckets: &HashMap<Uuid, Bucket>) { pub fn print_key_info(key: &Key, relevant_buckets: &HashMap<Uuid, Bucket>) {
let bucket_global_aliases = |b: &Uuid| { let bucket_global_aliases = |b: &Uuid| {
if let Some(bucket) = relevant_buckets.get(b) { if let Some(bucket) = relevant_buckets.get(b) {
@ -99,7 +139,7 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap<String, Key>)
.get(key_id) .get(key_id)
.map(|k| k.name.get().as_str()) .map(|k| k.name.get().as_str())
.unwrap_or(""); .unwrap_or("");
table.push(format!("\t{}\t{} ({})", alias, key_id, key_name)); table.push(format!("\t{} ({})\t{}", key_id, key_name, alias));
} }
} }
format_table(table); format_table(table);
@ -115,7 +155,7 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap<String, Key>)
.map(|k| k.name.get().as_str()) .map(|k| k.name.get().as_str())
.unwrap_or(""); .unwrap_or("");
table.push(format!( table.push(format!(
"\t{}{}{}\t{} ({})", "\t{}{}{}\t{}\t{}",
rflag, wflag, oflag, k, key_name rflag, wflag, oflag, k, key_name
)); ));
} }

4
src/garage/main.rs
View File

@ -139,8 +139,8 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into()); let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into());
match cli_command_dispatch(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await { match cli_command_dispatch(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await {
Err(HelperError::Internal(i)) => Err(i), Err(HelperError::Internal(i)) => Err(Error::Message(format!("Internal error: {}", i))),
Err(HelperError::BadRequest(b)) => Err(Error::Message(format!("bad request: {}", b))), Err(HelperError::BadRequest(b)) => Err(Error::Message(b)),
Ok(x) => Ok(x), Ok(x) => Ok(x),
} }
} }

26
src/model/bucket_table.rs
View File

@ -105,15 +105,29 @@ impl Bucket {
crdt::Deletable::Present(state) => state.authorized_keys.items(), crdt::Deletable::Present(state) => state.authorized_keys.items(),
} }
} }
pub fn aliases(&self) -> &[(String, u64, bool)] {
match &self.state {
crdt::Deletable::Deleted => &[],
crdt::Deletable::Present(state) => state.aliases.items(),
}
} }
impl Entry<Uuid, EmptyKey> for Bucket { pub fn local_aliases(&self) -> &[((String, String), u64, bool)] {
fn partition_key(&self) -> &Uuid { match &self.state {
&self.id crdt::Deletable::Deleted => &[],
crdt::Deletable::Present(state) => state.local_aliases.items(),
} }
fn sort_key(&self) -> &EmptyKey { }
}
impl Entry<EmptyKey, Uuid> for Bucket {
fn partition_key(&self) -> &EmptyKey {
&EmptyKey &EmptyKey
} }
fn sort_key(&self) -> &Uuid {
&self.id
}
} }
impl Crdt for Bucket { impl Crdt for Bucket {
@ -127,8 +141,8 @@ pub struct BucketTable;
impl TableSchema for BucketTable { impl TableSchema for BucketTable {
const TABLE_NAME: &'static str = "bucket_v2"; const TABLE_NAME: &'static str = "bucket_v2";
type P = Uuid; type P = EmptyKey;
type S = EmptyKey; type S = Uuid;
type E = Bucket; type E = Bucket;
type Filter = DeletedFilter; type Filter = DeletedFilter;

6
src/model/helper/bucket.rs
View File

@ -36,7 +36,7 @@ impl<'a> BucketHelper<'a> {
Ok(self Ok(self
.0 .0
.bucket_table .bucket_table
.get(&bucket_id, &EmptyKey) .get(&EmptyKey, &bucket_id)
.await? .await?
.filter(|x| !x.state.is_deleted()) .filter(|x| !x.state.is_deleted())
.map(|_| bucket_id)) .map(|_| bucket_id))
@ -58,7 +58,7 @@ impl<'a> BucketHelper<'a> {
Ok(self Ok(self
.0 .0
.bucket_table .bucket_table
.get(&bucket_id, &EmptyKey) .get(&EmptyKey, &bucket_id)
.await? .await?
.ok_or_message(format!("Bucket {:?} does not exist", bucket_id))?) .ok_or_message(format!("Bucket {:?} does not exist", bucket_id))?)
} }
@ -70,7 +70,7 @@ impl<'a> BucketHelper<'a> {
pub async fn get_existing_bucket(&self, bucket_id: Uuid) -> Result<Bucket, Error> { pub async fn get_existing_bucket(&self, bucket_id: Uuid) -> Result<Bucket, Error> {
self.0 self.0
.bucket_table .bucket_table
.get(&bucket_id, &EmptyKey) .get(&EmptyKey, &bucket_id)
.await? .await?
.filter(|b| !b.is_deleted()) .filter(|b| !b.is_deleted())
.ok_or_bad_request(format!( .ok_or_bad_request(format!(

2
src/web/web_server.rs
View File

@ -93,7 +93,7 @@ async fn serve_file(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<
// Check bucket isn't deleted and has website access enabled // Check bucket isn't deleted and has website access enabled
let _: Bucket = garage let _: Bucket = garage
.bucket_table .bucket_table
.get(&bucket_id, &EmptyKey) .get(&EmptyKey, &bucket_id)
.await? .await?
.filter(|b| { .filter(|b| {
b.state b.state