micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add test framework for arbitraty S3 requests

Browse Source 
and implement some basic test with it
This commit is contained in:
trinity-1686a 2022-03-11 17:35:08 +01:00 committed by Gitea
parent e934934f14
commit 1eb7fdb08f
14 changed files with 516 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
Cargo.lock generated
View File

@ -80,9 +80,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]] [[package]]
name = "aws-endpoint" name = "aws-endpoint"
version = "0.6.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6" checksum = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"
dependencies = [ dependencies = [
"aws-smithy-http", "aws-smithy-http",
"aws-types", "aws-types",
@ -93,9 +93,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-http" name = "aws-http"
version = "0.6.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1" checksum = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"
dependencies = [ dependencies = [
"aws-smithy-http", "aws-smithy-http",
"aws-smithy-types", "aws-smithy-types",
@ -108,9 +108,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-sdk-s3" name = "aws-sdk-s3"
version = "0.6.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9" checksum = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"
dependencies = [ dependencies = [
"aws-endpoint", "aws-endpoint",
"aws-http", "aws-http",
@ -133,9 +133,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-sig-auth" name = "aws-sig-auth"
version = "0.6.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc" checksum = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"
dependencies = [ dependencies = [
"aws-sigv4", "aws-sigv4",
"aws-smithy-eventstream", "aws-smithy-eventstream",
@ -148,9 +148,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-sigv4" name = "aws-sigv4"
version = "0.6.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c" checksum = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"
dependencies = [ dependencies = [
"aws-smithy-eventstream", "aws-smithy-eventstream",
"aws-smithy-http", "aws-smithy-http",
@ -168,9 +168,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-smithy-async" name = "aws-smithy-async"
version = "0.36.0" version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e" checksum = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"
dependencies = [ dependencies = [
"futures-util", "futures-util",
"pin-project-lite", "pin-project-lite",
@ -180,9 +180,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-smithy-client" name = "aws-smithy-client"
version = "0.36.0" version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb" checksum = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"
dependencies = [ dependencies = [
"aws-smithy-async", "aws-smithy-async",
"aws-smithy-http", "aws-smithy-http",
@ -204,9 +204,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-smithy-eventstream" name = "aws-smithy-eventstream"
version = "0.36.0" version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9" checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"
dependencies = [ dependencies = [
"aws-smithy-types", "aws-smithy-types",
"bytes 1.1.0", "bytes 1.1.0",
@ -215,9 +215,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-smithy-http" name = "aws-smithy-http"
version = "0.36.0" version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696" checksum = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"
dependencies = [ dependencies = [
"aws-smithy-eventstream", "aws-smithy-eventstream",
"aws-smithy-types", "aws-smithy-types",
@ -236,9 +236,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-smithy-http-tower" name = "aws-smithy-http-tower"
version = "0.36.0" version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d" checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"
dependencies = [ dependencies = [
"aws-smithy-http", "aws-smithy-http",
"bytes 1.1.0", "bytes 1.1.0",
@ -251,9 +251,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-smithy-types" name = "aws-smithy-types"
version = "0.36.0" version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f" checksum = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"
dependencies = [ dependencies = [
"itoa", "itoa",
"num-integer", "num-integer",
@ -263,9 +263,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-smithy-xml" name = "aws-smithy-xml"
version = "0.36.0" version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4" checksum = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"
dependencies = [ dependencies = [
"thiserror", "thiserror",
"xmlparser", "xmlparser",
@ -273,11 +273,12 @@ dependencies = [
[[package]] [[package]]
name = "aws-types" name = "aws-types"
version = "0.6.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142" checksum = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"
dependencies = [ dependencies = [
"aws-smithy-async", "aws-smithy-async",
"aws-smithy-client",
"aws-smithy-types", "aws-smithy-types",
"rustc_version", "rustc_version",
"tracing", "tracing",
@ -822,6 +823,7 @@ dependencies = [
"async-trait", "async-trait",
"aws-sdk-s3", "aws-sdk-s3",
"bytes 1.1.0", "bytes 1.1.0",
"chrono",
"futures", "futures",
"futures-util", "futures-util",
"garage_admin", "garage_admin",
@ -833,6 +835,7 @@ dependencies = [
"garage_web", "garage_web",
"git-version", "git-version",
"hex", "hex",
"hmac",
"http", "http",
"hyper", "hyper",
"kuska-sodiumoxide", "kuska-sodiumoxide",
@ -842,6 +845,7 @@ dependencies = [
"rmp-serde 0.15.5", "rmp-serde 0.15.5",
"serde", "serde",
"serde_bytes", "serde_bytes",
"sha2",
"sled", "sled",
"static_init", "static_init",
"structopt", "structopt",

150
Cargo.nix
View File

@ -150,29 +150,29 @@ in
src = fetchCratesIo { inherit name version; sha256 = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"; }; src = fetchCratesIo { inherit name version; sha256 = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-endpoint"; name = "aws-endpoint";
version = "0.6.0"; version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"; }; src = fetchCratesIo { inherit name version; sha256 = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"; };
dependencies = { dependencies = {
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; }; aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
regex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; }; regex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; };
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-http"; name = "aws-http";
version = "0.6.0"; version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"; }; src = fetchCratesIo { inherit name version; sha256 = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"; };
dependencies = { dependencies = {
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; }; aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
@ -180,29 +180,29 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-sdk-s3"; name = "aws-sdk-s3";
version = "0.6.0"; version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"; }; src = fetchCratesIo { inherit name version; sha256 = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"; };
features = builtins.concatLists [ features = builtins.concatLists [
[ "default" ] [ "default" ]
[ "rt-tokio" ] [ "rt-tokio" ]
[ "rustls" ] [ "rustls" ]
]; ];
dependencies = { dependencies = {
aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" { inherit profileName; }; aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" { inherit profileName; };
aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" { inherit profileName; }; aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" { inherit profileName; };
aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" { inherit profileName; }; aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" { inherit profileName; };
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; }; aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; };
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; }; aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" { inherit profileName; }; aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; };
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; }; aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; }; aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" { inherit profileName; }; aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; }; aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md5."0.7.0" { inherit profileName; }; md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md5."0.7.0" { inherit profileName; };
@ -211,31 +211,31 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-sig-auth"; name = "aws-sig-auth";
version = "0.6.0"; version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"; }; src = fetchCratesIo { inherit name version; sha256 = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"; };
features = builtins.concatLists [ features = builtins.concatLists [
[ "aws-smithy-eventstream" ] [ "aws-smithy-eventstream" ]
[ "sign-eventstream" ] [ "sign-eventstream" ]
]; ];
dependencies = { dependencies = {
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; }; aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; };
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; }; aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; }; aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-sigv4"; name = "aws-sigv4";
version = "0.6.0"; version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"; }; src = fetchCratesIo { inherit name version; sha256 = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"; };
features = builtins.concatLists [ features = builtins.concatLists [
[ "aws-smithy-eventstream" ] [ "aws-smithy-eventstream" ]
[ "bytes" ] [ "bytes" ]
@ -247,8 +247,8 @@ in
[ "sign-http" ] [ "sign-http" ]
]; ];
dependencies = { dependencies = {
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; }; aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }; form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; };
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
@ -262,11 +262,11 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-async"; name = "aws-smithy-async";
version = "0.36.0"; version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"; }; src = fetchCratesIo { inherit name version; sha256 = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"; };
features = builtins.concatLists [ features = builtins.concatLists [
[ "rt-tokio" ] [ "rt-tokio" ]
]; ];
@ -278,11 +278,11 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-client"; name = "aws-smithy-client";
version = "0.36.0"; version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"; }; src = fetchCratesIo { inherit name version; sha256 = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"; };
features = builtins.concatLists [ features = builtins.concatLists [
[ "client-hyper" ] [ "client-hyper" ]
[ "hyper" ] [ "hyper" ]
@ -292,10 +292,10 @@ in
[ "rustls" ] [ "rustls" ]
]; ];
dependencies = { dependencies = {
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; }; aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; }; aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
fastrand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.7.0" { inherit profileName; }; fastrand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.7.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
@ -311,23 +311,23 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-eventstream"; name = "aws-smithy-eventstream";
version = "0.36.0"; version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"; }; src = fetchCratesIo { inherit name version; sha256 = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"; };
dependencies = { dependencies = {
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
crc32fast = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }; crc32fast = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; };
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-http"; name = "aws-smithy-http";
version = "0.36.0"; version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"; }; src = fetchCratesIo { inherit name version; sha256 = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"; };
features = builtins.concatLists [ features = builtins.concatLists [
[ "aws-smithy-eventstream" ] [ "aws-smithy-eventstream" ]
[ "event-stream" ] [ "event-stream" ]
@ -336,8 +336,8 @@ in
[ "tokio-util" ] [ "tokio-util" ]
]; ];
dependencies = { dependencies = {
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; }; aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
bytes_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.1" { inherit profileName; }; bytes_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.1" { inherit profileName; };
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
@ -352,13 +352,13 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-http-tower"; name = "aws-smithy-http-tower";
version = "0.36.0"; version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"; }; src = fetchCratesIo { inherit name version; sha256 = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"; };
dependencies = { dependencies = {
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; }; http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; };
@ -368,11 +368,11 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-types"; name = "aws-smithy-types";
version = "0.36.0"; version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"; }; src = fetchCratesIo { inherit name version; sha256 = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"; };
dependencies = { dependencies = {
itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; }; itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; };
num_integer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }; num_integer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; };
@ -381,25 +381,26 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-xml"; name = "aws-smithy-xml";
version = "0.36.0"; version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"; }; src = fetchCratesIo { inherit name version; sha256 = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"; };
dependencies = { dependencies = {
thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
xmlparser = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" { inherit profileName; }; xmlparser = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" { inherit profileName; };
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-types"; name = "aws-types";
version = "0.6.0"; version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"; }; src = fetchCratesIo { inherit name version; sha256 = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"; };
dependencies = { dependencies = {
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; }; aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
zeroize = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.3" { inherit profileName; }; zeroize = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.3" { inherit profileName; };
}; };
@ -1196,9 +1197,12 @@ in
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
}; };
devDependencies = { devDependencies = {
aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" { inherit profileName; }; aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" { inherit profileName; };
chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; };
sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; };
static_init = rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.2" { inherit profileName; }; static_init = rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.2" { inherit profileName; };
}; };
}); });

3
src/api/lib.rs
View File

@ -10,7 +10,8 @@ mod encoding;
mod api_server; mod api_server;
pub use api_server::run_api_server; pub use api_server::run_api_server;
mod signature; /// This mode is public only to help testing. Don't expect stability here
pub mod signature;
pub mod helpers; pub mod helpers;
mod s3_bucket; mod s3_bucket;

14
src/api/signature/payload.rs
View File

@ -51,8 +51,7 @@ pub async fn check_payload_signature(
let canonical_request = canonical_request( let canonical_request = canonical_request(
request.method(), request.method(),
request.uri().path(), request.uri(),
&canonical_query_string(request.uri()),
&headers, &headers,
&authorization.signed_headers, &authorization.signed_headers,
&authorization.content_sha256, &authorization.content_sha256,
@ -215,7 +214,7 @@ fn parse_credential(cred: &str) -> Result<(String, String), Error> {
)) ))
} }
fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String { pub fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
let mut hasher = Sha256::default(); let mut hasher = Sha256::default();
hasher.update(canonical_req.as_bytes()); hasher.update(canonical_req.as_bytes());
[ [
@ -227,18 +226,17 @@ fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &
.join("\n") .join("\n")
} }
fn canonical_request( pub fn canonical_request(
method: &Method, method: &Method,
url_path: &str, uri: &hyper::Uri,
canonical_query_string: &str,
headers: &HashMap<String, String>, headers: &HashMap<String, String>,
signed_headers: &str, signed_headers: &str,
content_sha256: &str, content_sha256: &str,
) -> String { ) -> String {
[ [
method.as_str(), method.as_str(),
url_path, &uri.path().to_string(),
canonical_query_string, &canonical_query_string(uri),
&canonical_header_string(headers, signed_headers), &canonical_header_string(headers, signed_headers),
"", "",
signed_headers, signed_headers,

5
src/garage/Cargo.toml
View File

@ -55,8 +55,11 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
netapp = "0.4.1" netapp = "0.4.1"
[dev-dependencies] [dev-dependencies]
aws-sdk-s3 = "0.6" aws-sdk-s3 = "0.8"
chrono = "0.4"
http = "0.2" http = "0.2"
hmac = "0.10"
hyper = { version = "0.14", features = ["client", "http1", "runtime"] } hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
sha2 = "0.9"
static_init = "1.0" static_init = "1.0"

280
src/garage/tests/common/custom_requester.rs Normal file
View File

@ -0,0 +1,280 @@
#![allow(dead_code)]
use std::collections::HashMap;
use std::convert::TryFrom;
use chrono::{offset::Utc, DateTime};
use hmac::{Hmac, Mac};
use hyper::client::HttpConnector;
use hyper::{Body, Client, Method, Request, Response, Uri};
use super::garage::{Instance, Key};
use garage_api::signature;
/// You should ever only use this to send requests AWS sdk won't send,
/// like to reproduce behavior of unusual implementations found to be
/// problematic.
pub struct CustomRequester {
key: Key,
uri: Uri,
client: Client<HttpConnector>,
}
impl CustomRequester {
pub fn new(instance: &Instance) -> Self {
/*
let credentials = Credentials::new(
&instance.key.id,
&instance.key.secret,
None,
None,
"garage-integ-test",
);
let endpoint = Endpoint::immutable(instance.uri());
*/
CustomRequester {
key: instance.key.clone(),
uri: instance.uri(),
client: Client::new(),
}
}
pub fn builder(&self, bucket: String) -> RequestBuilder<'_> {
RequestBuilder {
requester: self,
bucket,
method: Method::GET,
path: String::new(),
query_params: HashMap::new(),
signed_headers: HashMap::new(),
unsigned_headers: HashMap::new(),
body: Vec::new(),
body_signature: BodySignature::Classic,
vhost_style: false,
}
}
/*
pub async fn request(&self, method: &str, path: String, headers: &HashMap<String, (bool, String)>, body: &[u8], vhost_style: bool) -> hyper::Result<Response<Body>> {
let request = Request::builder()
.method(
self.client.request(todo!()).await
}
*/
}
pub struct RequestBuilder<'a> {
requester: &'a CustomRequester,
bucket: String,
method: Method,
path: String,
query_params: HashMap<String, Option<String>>,
signed_headers: HashMap<String, String>,
unsigned_headers: HashMap<String, String>,
body: Vec<u8>,
body_signature: BodySignature,
vhost_style: bool,
}
impl<'a> RequestBuilder<'a> {
pub fn method(&mut self, method: Method) -> &mut Self {
self.method = method;
self
}
pub fn path(&mut self, path: String) -> &mut Self {
self.path = path;
self
}
pub fn query_params(&mut self, query_params: HashMap<String, Option<String>>) -> &mut Self {
self.query_params = query_params;
self
}
pub fn signed_headers(&mut self, signed_headers: HashMap<String, String>) -> &mut Self {
self.signed_headers = signed_headers;
self
}
pub fn unsigned_headers(&mut self, unsigned_headers: HashMap<String, String>) -> &mut Self {
self.unsigned_headers = unsigned_headers;
self
}
pub fn body(&mut self, body: Vec<u8>) -> &mut Self {
self.body = body;
self
}
pub fn body_signature(&mut self, body_signature: BodySignature) -> &mut Self {
self.body_signature = body_signature;
self
}
pub fn vhost_style(&mut self, vhost_style: bool) -> &mut Self {
self.vhost_style = vhost_style;
self
}
pub async fn send(&mut self) -> hyper::Result<Response<Body>> {
// TODO this is a bit incorrect in that path and query params should be url-encoded and
// aren't, but this is good enought for now.
let query = query_param_to_string(&self.query_params);
let (host, path) = if self.vhost_style {
(
format!("{}.s3.garage", self.bucket),
format!("{}{}", self.path, query),
)
} else {
(
"s3.garage".to_owned(),
format!("{}/{}{}", self.bucket, self.path, query),
)
};
let uri = format!("{}{}", self.requester.uri, path);
let now = Utc::now();
let scope = signature::compute_scope(&now, super::REGION.as_ref());
let mut signer = signature::signing_hmac(
&now,
&self.requester.key.secret,
super::REGION.as_ref(),
"s3",
)
.unwrap();
let streaming_signer = signer.clone();
let mut all_headers = self.signed_headers.clone();
let date = now.format(signature::LONG_DATETIME).to_string();
all_headers.insert("x-amz-date".to_owned(), date);
all_headers.insert("host".to_owned(), host);
let body_sha = match self.body_signature {
BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(),
BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)),
BodySignature::Streaming(size) => {
all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned());
all_headers.insert(
"x-amz-decoded-content-length".to_owned(),
self.body.len().to_string(),
);
// this is a pretty lazy and inefficient way to do it, but it's enought for
// test code.
all_headers.insert(
"content-length".to_owned(),
to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "")
.len()
.to_string(),
);
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned()
}
};
all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone());
let mut signed_headers = all_headers
.iter()
.map(|(k, _)| k.as_ref())
.collect::<Vec<&str>>();
signed_headers.sort();
let signed_headers = signed_headers.join(";");
all_headers.extend(self.unsigned_headers.clone());
let canonical_request = signature::payload::canonical_request(
&self.method,
&Uri::try_from(&uri).unwrap(),
&all_headers,
&signed_headers,
&body_sha,
);
let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request);
signer.update(string_to_sign.as_bytes());
let signature = hex::encode(signer.finalize().into_bytes());
let authorization = format!(
"AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}",
self.requester.key.id, scope, signed_headers, signature
);
all_headers.insert("authorization".to_owned(), authorization);
let mut request = Request::builder();
for (k, v) in all_headers {
request = request.header(k, v);
}
let body = if let BodySignature::Streaming(size) = self.body_signature {
to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope)
} else {
self.body.clone()
};
let request = request
.uri(uri)
.method(self.method.clone())
.body(Body::from(body))
.unwrap();
self.requester.client.request(request).await
}
}
pub enum BodySignature {
Unsigned,
Classic,
Streaming(usize),
}
fn query_param_to_string(params: &HashMap<String, Option<String>>) -> String {
if params.is_empty() {
return String::new();
}
"?".to_owned()
+ &params
.iter()
.map(|(k, v)| {
if let Some(v) = v {
format!("{}={}", k, v)
} else {
k.clone()
}
})
.collect::<Vec<String>>()
.join("&")
}
fn to_streaming_body(
body: &[u8],
chunk_size: usize,
mut seed: String,
hasher: Hmac<sha2::Sha256>,
now: DateTime<Utc>,
scope: &str,
) -> Vec<u8> {
const SHA_NULL: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
let now = now.format(signature::LONG_DATETIME).to_string();
let mut res = Vec::with_capacity(body.len());
for chunk in body.chunks(chunk_size).chain(std::iter::once(&[][..])) {
let to_sign = format!(
"AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}",
now,
scope,
seed,
SHA_NULL,
hex::encode(garage_util::data::sha256sum(chunk))
);
let mut hasher = hasher.clone();
hasher.update(to_sign.as_bytes());
seed = hex::encode(hasher.finalize().into_bytes());
let header = format!("{:x};chunk-signature={}\r\n", chunk.len(), seed);
res.extend_from_slice(header.as_bytes());
res.extend_from_slice(chunk);
res.extend_from_slice(b"\r\n");
}
res
}

2
src/garage/tests/common/garage.rs
View File

@ -11,7 +11,7 @@ pub const DEFAULT_PORT: u16 = 49995;
static GARAGE_TEST_SECRET: &str = static GARAGE_TEST_SECRET: &str =
"c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44"; "c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44";
#[derive(Debug, Default)] #[derive(Debug, Default, Clone)]
pub struct Key { pub struct Key {
pub name: String, pub name: String,
pub id: String, pub id: String,

11
src/garage/tests/common/mod.rs
View File

@ -5,22 +5,31 @@ use ext::*;
pub mod macros; pub mod macros;
pub mod client; pub mod client;
pub mod custom_requester;
pub mod ext; pub mod ext;
pub mod garage; pub mod garage;
use custom_requester::CustomRequester;
const REGION: Region = Region::from_static("garage-integ-test"); const REGION: Region = Region::from_static("garage-integ-test");
pub struct Context { pub struct Context {
pub garage: &'static garage::Instance, pub garage: &'static garage::Instance,
pub client: Client, pub client: Client,
pub custom_request: CustomRequester,
} }
impl Context { impl Context {
fn new() -> Self { fn new() -> Self {
let garage = garage::instance(); let garage = garage::instance();
let client = client::build_client(garage); let client = client::build_client(garage);
let custom_request = CustomRequester::new(garage);
Context { garage, client } Context {
garage,
client,
custom_request,
}
} }
/// Create an unique bucket with a random suffix. /// Create an unique bucket with a random suffix.

1
src/garage/tests/lib.rs
View File

@ -7,4 +7,5 @@ mod list;
mod multipart; mod multipart;
mod objects; mod objects;
mod simple; mod simple;
mod streaming_signature;
mod website; mod website;

2
src/garage/tests/multipart.rs
View File

@ -1,6 +1,6 @@
use crate::common; use crate::common;
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart}; use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::ByteStream; use aws_sdk_s3::types::ByteStream;
const SZ_5MB: usize = 5 * 1024 * 1024; const SZ_5MB: usize = 5 * 1024 * 1024;
const SZ_10MB: usize = 10 * 1024 * 1024; const SZ_10MB: usize = 10 * 1024 * 1024;

2
src/garage/tests/objects.rs
View File

@ -1,6 +1,6 @@
use crate::common; use crate::common;
use aws_sdk_s3::model::{Delete, ObjectIdentifier}; use aws_sdk_s3::model::{Delete, ObjectIdentifier};
use aws_sdk_s3::ByteStream; use aws_sdk_s3::types::ByteStream;
const STD_KEY: &str = "hello world"; const STD_KEY: &str = "hello world";
const CTRL_KEY: &str = "\x00\x01\x02\x00"; const CTRL_KEY: &str = "\x00\x01\x02\x00";

2
src/garage/tests/simple.rs
View File

@ -2,7 +2,7 @@ use crate::common;
#[tokio::test] #[tokio::test]
async fn test_simple() { async fn test_simple() {
use aws_sdk_s3::ByteStream; use aws_sdk_s3::types::ByteStream;
let ctx = common::context(); let ctx = common::context();
let bucket = ctx.create_bucket("test-simple"); let bucket = ctx.create_bucket("test-simple");

101
src/garage/tests/streaming_signature.rs Normal file
View File

@ -0,0 +1,101 @@
use std::collections::HashMap;
use crate::common;
use common::custom_requester::BodySignature;
use hyper::Method;
const STD_KEY: &str = "hello-world";
//const CTRL_KEY: &str = "\x00\x01\x02\x00";
const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
#[tokio::test]
async fn test_putobject_streaming() {
let ctx = common::context();
let bucket = ctx.create_bucket("putobject-streaming");
{
// Send an empty object (can serve as a directory marker)
// with a content type
let etag = "\"d41d8cd98f00b204e9800998ecf8427e\"";
let content_type = "text/csv";
let mut headers = HashMap::new();
headers.insert("content-type".to_owned(), content_type.to_owned());
let _ = ctx
.custom_request
.builder(bucket.clone())
.method(Method::PUT)
.path(STD_KEY.to_owned())
.unsigned_headers(headers)
.vhost_style(true)
.body(vec![])
.body_signature(BodySignature::Streaming(10))
.send()
.await
.unwrap();
// assert_eq!(r.e_tag.unwrap().as_str(), etag);
// We return a version ID here
// We should check if Amazon is returning one when versioning is not enabled
// assert!(r.version_id.is_some());
//let _version = r.version_id.unwrap();
let o = ctx
.client
.get_object()
.bucket(&bucket)
.key(STD_KEY)
.send()
.await
.unwrap();
assert_bytes_eq!(o.body, b"");
assert_eq!(o.e_tag.unwrap(), etag);
// We do not return version ID
// We should check if Amazon is returning one when versioning is not enabled
// assert_eq!(o.version_id.unwrap(), _version);
assert_eq!(o.content_type.unwrap(), content_type);
assert!(o.last_modified.is_some());
assert_eq!(o.content_length, 0);
assert_eq!(o.parts_count, 0);
assert_eq!(o.tag_count, 0);
}
{
let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
let _ = ctx
.custom_request
.builder(bucket.clone())
.method(Method::PUT)
//.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this
//fail
.path("abc".to_owned())
.vhost_style(true)
.body(BODY.to_vec())
.body_signature(BodySignature::Streaming(16))
.send()
.await
.unwrap();
// assert_eq!(r.e_tag.unwrap().as_str(), etag);
// assert!(r.version_id.is_some());
let o = ctx
.client
.get_object()
.bucket(&bucket)
//.key(CTRL_KEY)
.key("abc")
.send()
.await
.unwrap();
assert_bytes_eq!(o.body, BODY);
assert_eq!(o.e_tag.unwrap(), etag);
assert!(o.last_modified.is_some());
assert_eq!(o.content_length, 62);
assert_eq!(o.parts_count, 0);
assert_eq!(o.tag_count, 0);
}
}

2
src/garage/tests/website.rs
View File

@ -2,7 +2,7 @@ use crate::common;
use crate::common::ext::*; use crate::common::ext::*;
use aws_sdk_s3::{ use aws_sdk_s3::{
model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
ByteStream, types::ByteStream,
}; };
use http::Request; use http::Request;
use hyper::{ use hyper::{