add test framework for arbitraty S3 requests
and implement some basic test with it
This commit is contained in:
trinity-1686a
Gitea
parent
e934934f14
commit
1eb7fdb08f
56
Cargo.lock
generated
56
Cargo.lock
generated
@ -80,9 +80,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "aws-endpoint"
|
||||
version = "0.6.0"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"
|
||||
checksum = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"
|
||||
dependencies = [
|
||||
"aws-smithy-http",
|
||||
"aws-types",
|
||||
@ -93,9 +93,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-http"
|
||||
version = "0.6.0"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"
|
||||
checksum = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"
|
||||
dependencies = [
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-types",
|
||||
@ -108,9 +108,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "0.6.0"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"
|
||||
checksum = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"
|
||||
dependencies = [
|
||||
"aws-endpoint",
|
||||
"aws-http",
|
||||
@ -133,9 +133,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sig-auth"
|
||||
version = "0.6.0"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"
|
||||
checksum = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"
|
||||
dependencies = [
|
||||
"aws-sigv4",
|
||||
"aws-smithy-eventstream",
|
||||
@ -148,9 +148,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "0.6.0"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"
|
||||
checksum = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-http",
|
||||
@ -168,9 +168,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-async"
|
||||
version = "0.36.0"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"
|
||||
checksum = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"pin-project-lite",
|
||||
@ -180,9 +180,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-client"
|
||||
version = "0.36.0"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"
|
||||
checksum = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@ -204,9 +204,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-eventstream"
|
||||
version = "0.36.0"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"
|
||||
checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes 1.1.0",
|
||||
@ -215,9 +215,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.36.0"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"
|
||||
checksum = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-types",
|
||||
@ -236,9 +236,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-tower"
|
||||
version = "0.36.0"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"
|
||||
checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"
|
||||
dependencies = [
|
||||
"aws-smithy-http",
|
||||
"bytes 1.1.0",
|
||||
@ -251,9 +251,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "0.36.0"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"
|
||||
checksum = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"num-integer",
|
||||
@ -263,9 +263,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.36.0"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"
|
||||
checksum = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"
|
||||
dependencies = [
|
||||
"thiserror",
|
||||
"xmlparser",
|
||||
@ -273,11 +273,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-types"
|
||||
version = "0.6.0"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"
|
||||
checksum = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-client",
|
||||
"aws-smithy-types",
|
||||
"rustc_version",
|
||||
"tracing",
|
||||
@ -822,6 +823,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"aws-sdk-s3",
|
||||
"bytes 1.1.0",
|
||||
"chrono",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_admin",
|
||||
@ -833,6 +835,7 @@ dependencies = [
|
||||
"garage_web",
|
||||
"git-version",
|
||||
"hex",
|
||||
"hmac",
|
||||
"http",
|
||||
"hyper",
|
||||
"kuska-sodiumoxide",
|
||||
@ -842,6 +845,7 @@ dependencies = [
|
||||
"rmp-serde 0.15.5",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"sha2",
|
||||
"sled",
|
||||
"static_init",
|
||||
"structopt",
|
||||
|
||||
150
Cargo.nix
150
Cargo.nix
@ -150,29 +150,29 @@ in
|
||||
src = fetchCratesIo { inherit name version; sha256 = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"; };
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-endpoint";
|
||||
version = "0.6.0";
|
||||
version = "0.8.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"; };
|
||||
dependencies = {
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
regex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; };
|
||||
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-http";
|
||||
version = "0.6.0";
|
||||
version = "0.8.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"; };
|
||||
dependencies = {
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
|
||||
percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
|
||||
@ -180,29 +180,29 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-sdk-s3";
|
||||
version = "0.6.0";
|
||||
version = "0.8.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"; };
|
||||
features = builtins.concatLists [
|
||||
[ "default" ]
|
||||
[ "rt-tokio" ]
|
||||
[ "rustls" ]
|
||||
];
|
||||
dependencies = {
|
||||
aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" { inherit profileName; };
|
||||
aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" { inherit profileName; };
|
||||
aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" { inherit profileName; };
|
||||
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; };
|
||||
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
|
||||
aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" { inherit profileName; };
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
|
||||
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
|
||||
aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
|
||||
aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" { inherit profileName; };
|
||||
aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" { inherit profileName; };
|
||||
aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" { inherit profileName; };
|
||||
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; };
|
||||
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
|
||||
aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; };
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md5."0.7.0" { inherit profileName; };
|
||||
@ -211,31 +211,31 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-sig-auth";
|
||||
version = "0.6.0";
|
||||
version = "0.8.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"; };
|
||||
features = builtins.concatLists [
|
||||
[ "aws-smithy-eventstream" ]
|
||||
[ "sign-eventstream" ]
|
||||
];
|
||||
dependencies = {
|
||||
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; };
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
|
||||
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; };
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
|
||||
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-sigv4";
|
||||
version = "0.6.0";
|
||||
version = "0.8.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"; };
|
||||
features = builtins.concatLists [
|
||||
[ "aws-smithy-eventstream" ]
|
||||
[ "bytes" ]
|
||||
@ -247,8 +247,8 @@ in
|
||||
[ "sign-http" ]
|
||||
];
|
||||
dependencies = {
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
@ -262,11 +262,11 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-smithy-async";
|
||||
version = "0.36.0";
|
||||
version = "0.38.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"; };
|
||||
features = builtins.concatLists [
|
||||
[ "rt-tokio" ]
|
||||
];
|
||||
@ -278,11 +278,11 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-smithy-client";
|
||||
version = "0.36.0";
|
||||
version = "0.38.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"; };
|
||||
features = builtins.concatLists [
|
||||
[ "client-hyper" ]
|
||||
[ "hyper" ]
|
||||
@ -292,10 +292,10 @@ in
|
||||
[ "rustls" ]
|
||||
];
|
||||
dependencies = {
|
||||
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
|
||||
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
|
||||
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
fastrand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.7.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
@ -311,23 +311,23 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-smithy-eventstream";
|
||||
version = "0.36.0";
|
||||
version = "0.38.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"; };
|
||||
dependencies = {
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
crc32fast = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-smithy-http";
|
||||
version = "0.36.0";
|
||||
version = "0.38.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"; };
|
||||
features = builtins.concatLists [
|
||||
[ "aws-smithy-eventstream" ]
|
||||
[ "event-stream" ]
|
||||
@ -336,8 +336,8 @@ in
|
||||
[ "tokio-util" ]
|
||||
];
|
||||
dependencies = {
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.1" { inherit profileName; };
|
||||
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
@ -352,13 +352,13 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-smithy-http-tower";
|
||||
version = "0.36.0";
|
||||
version = "0.38.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"; };
|
||||
dependencies = {
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; };
|
||||
@ -368,11 +368,11 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-smithy-types";
|
||||
version = "0.36.0";
|
||||
version = "0.38.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"; };
|
||||
dependencies = {
|
||||
itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; };
|
||||
num_integer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; };
|
||||
@ -381,25 +381,26 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-smithy-xml";
|
||||
version = "0.36.0";
|
||||
version = "0.38.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"; };
|
||||
dependencies = {
|
||||
thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
|
||||
xmlparser = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aws-types";
|
||||
version = "0.6.0";
|
||||
version = "0.8.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"; };
|
||||
dependencies = {
|
||||
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
|
||||
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
|
||||
aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
|
||||
zeroize = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.3" { inherit profileName; };
|
||||
};
|
||||
@ -1196,9 +1197,12 @@ in
|
||||
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
|
||||
};
|
||||
devDependencies = {
|
||||
aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" { inherit profileName; };
|
||||
aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" { inherit profileName; };
|
||||
chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
|
||||
hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; };
|
||||
sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; };
|
||||
static_init = rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.2" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
@ -10,7 +10,8 @@ mod encoding;
|
||||
mod api_server;
|
||||
pub use api_server::run_api_server;
|
||||
|
||||
mod signature;
|
||||
/// This mode is public only to help testing. Don't expect stability here
|
||||
pub mod signature;
|
||||
|
||||
pub mod helpers;
|
||||
mod s3_bucket;
|
||||
|
||||
@ -51,8 +51,7 @@ pub async fn check_payload_signature(
|
||||
|
||||
let canonical_request = canonical_request(
|
||||
request.method(),
|
||||
request.uri().path(),
|
||||
&canonical_query_string(request.uri()),
|
||||
request.uri(),
|
||||
&headers,
|
||||
&authorization.signed_headers,
|
||||
&authorization.content_sha256,
|
||||
@ -215,7 +214,7 @@ fn parse_credential(cred: &str) -> Result<(String, String), Error> {
|
||||
))
|
||||
}
|
||||
|
||||
fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
|
||||
pub fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
|
||||
let mut hasher = Sha256::default();
|
||||
hasher.update(canonical_req.as_bytes());
|
||||
[
|
||||
@ -227,18 +226,17 @@ fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn canonical_request(
|
||||
pub fn canonical_request(
|
||||
method: &Method,
|
||||
url_path: &str,
|
||||
canonical_query_string: &str,
|
||||
uri: &hyper::Uri,
|
||||
headers: &HashMap<String, String>,
|
||||
signed_headers: &str,
|
||||
content_sha256: &str,
|
||||
) -> String {
|
||||
[
|
||||
method.as_str(),
|
||||
url_path,
|
||||
canonical_query_string,
|
||||
&uri.path().to_string(),
|
||||
&canonical_query_string(uri),
|
||||
&canonical_header_string(headers, signed_headers),
|
||||
"",
|
||||
signed_headers,
|
||||
|
||||
@ -55,8 +55,11 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
|
||||
netapp = "0.4.1"
|
||||
|
||||
[dev-dependencies]
|
||||
aws-sdk-s3 = "0.6"
|
||||
aws-sdk-s3 = "0.8"
|
||||
chrono = "0.4"
|
||||
http = "0.2"
|
||||
hmac = "0.10"
|
||||
hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
|
||||
sha2 = "0.9"
|
||||
|
||||
static_init = "1.0"
|
||||
|
||||
280
src/garage/tests/common/custom_requester.rs
Normal file
280
src/garage/tests/common/custom_requester.rs
Normal file
@ -0,0 +1,280 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use chrono::{offset::Utc, DateTime};
|
||||
use hmac::{Hmac, Mac};
|
||||
use hyper::client::HttpConnector;
|
||||
use hyper::{Body, Client, Method, Request, Response, Uri};
|
||||
|
||||
use super::garage::{Instance, Key};
|
||||
use garage_api::signature;
|
||||
|
||||
/// You should ever only use this to send requests AWS sdk won't send,
|
||||
/// like to reproduce behavior of unusual implementations found to be
|
||||
/// problematic.
|
||||
pub struct CustomRequester {
|
||||
key: Key,
|
||||
uri: Uri,
|
||||
client: Client<HttpConnector>,
|
||||
}
|
||||
|
||||
impl CustomRequester {
|
||||
pub fn new(instance: &Instance) -> Self {
|
||||
/*
|
||||
let credentials = Credentials::new(
|
||||
&instance.key.id,
|
||||
&instance.key.secret,
|
||||
None,
|
||||
None,
|
||||
"garage-integ-test",
|
||||
);
|
||||
let endpoint = Endpoint::immutable(instance.uri());
|
||||
*/
|
||||
CustomRequester {
|
||||
key: instance.key.clone(),
|
||||
uri: instance.uri(),
|
||||
client: Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn builder(&self, bucket: String) -> RequestBuilder<'_> {
|
||||
RequestBuilder {
|
||||
requester: self,
|
||||
bucket,
|
||||
method: Method::GET,
|
||||
path: String::new(),
|
||||
query_params: HashMap::new(),
|
||||
signed_headers: HashMap::new(),
|
||||
unsigned_headers: HashMap::new(),
|
||||
body: Vec::new(),
|
||||
body_signature: BodySignature::Classic,
|
||||
vhost_style: false,
|
||||
}
|
||||
}
|
||||
/*
|
||||
pub async fn request(&self, method: &str, path: String, headers: &HashMap<String, (bool, String)>, body: &[u8], vhost_style: bool) -> hyper::Result<Response<Body>> {
|
||||
let request = Request::builder()
|
||||
.method(
|
||||
self.client.request(todo!()).await
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
pub struct RequestBuilder<'a> {
|
||||
requester: &'a CustomRequester,
|
||||
bucket: String,
|
||||
method: Method,
|
||||
path: String,
|
||||
query_params: HashMap<String, Option<String>>,
|
||||
signed_headers: HashMap<String, String>,
|
||||
unsigned_headers: HashMap<String, String>,
|
||||
body: Vec<u8>,
|
||||
body_signature: BodySignature,
|
||||
vhost_style: bool,
|
||||
}
|
||||
|
||||
impl<'a> RequestBuilder<'a> {
|
||||
pub fn method(&mut self, method: Method) -> &mut Self {
|
||||
self.method = method;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn path(&mut self, path: String) -> &mut Self {
|
||||
self.path = path;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn query_params(&mut self, query_params: HashMap<String, Option<String>>) -> &mut Self {
|
||||
self.query_params = query_params;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn signed_headers(&mut self, signed_headers: HashMap<String, String>) -> &mut Self {
|
||||
self.signed_headers = signed_headers;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn unsigned_headers(&mut self, unsigned_headers: HashMap<String, String>) -> &mut Self {
|
||||
self.unsigned_headers = unsigned_headers;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn body(&mut self, body: Vec<u8>) -> &mut Self {
|
||||
self.body = body;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn body_signature(&mut self, body_signature: BodySignature) -> &mut Self {
|
||||
self.body_signature = body_signature;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn vhost_style(&mut self, vhost_style: bool) -> &mut Self {
|
||||
self.vhost_style = vhost_style;
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn send(&mut self) -> hyper::Result<Response<Body>> {
|
||||
// TODO this is a bit incorrect in that path and query params should be url-encoded and
|
||||
// aren't, but this is good enought for now.
|
||||
|
||||
let query = query_param_to_string(&self.query_params);
|
||||
let (host, path) = if self.vhost_style {
|
||||
(
|
||||
format!("{}.s3.garage", self.bucket),
|
||||
format!("{}{}", self.path, query),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"s3.garage".to_owned(),
|
||||
format!("{}/{}{}", self.bucket, self.path, query),
|
||||
)
|
||||
};
|
||||
let uri = format!("{}{}", self.requester.uri, path);
|
||||
|
||||
let now = Utc::now();
|
||||
let scope = signature::compute_scope(&now, super::REGION.as_ref());
|
||||
let mut signer = signature::signing_hmac(
|
||||
&now,
|
||||
&self.requester.key.secret,
|
||||
super::REGION.as_ref(),
|
||||
"s3",
|
||||
)
|
||||
.unwrap();
|
||||
let streaming_signer = signer.clone();
|
||||
|
||||
let mut all_headers = self.signed_headers.clone();
|
||||
|
||||
let date = now.format(signature::LONG_DATETIME).to_string();
|
||||
all_headers.insert("x-amz-date".to_owned(), date);
|
||||
all_headers.insert("host".to_owned(), host);
|
||||
|
||||
let body_sha = match self.body_signature {
|
||||
BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(),
|
||||
BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)),
|
||||
BodySignature::Streaming(size) => {
|
||||
all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned());
|
||||
all_headers.insert(
|
||||
"x-amz-decoded-content-length".to_owned(),
|
||||
self.body.len().to_string(),
|
||||
);
|
||||
// this is a pretty lazy and inefficient way to do it, but it's enought for
|
||||
// test code.
|
||||
all_headers.insert(
|
||||
"content-length".to_owned(),
|
||||
to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "")
|
||||
.len()
|
||||
.to_string(),
|
||||
);
|
||||
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned()
|
||||
}
|
||||
};
|
||||
all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone());
|
||||
|
||||
let mut signed_headers = all_headers
|
||||
.iter()
|
||||
.map(|(k, _)| k.as_ref())
|
||||
.collect::<Vec<&str>>();
|
||||
signed_headers.sort();
|
||||
let signed_headers = signed_headers.join(";");
|
||||
|
||||
all_headers.extend(self.unsigned_headers.clone());
|
||||
|
||||
let canonical_request = signature::payload::canonical_request(
|
||||
&self.method,
|
||||
&Uri::try_from(&uri).unwrap(),
|
||||
&all_headers,
|
||||
&signed_headers,
|
||||
&body_sha,
|
||||
);
|
||||
|
||||
let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request);
|
||||
|
||||
signer.update(string_to_sign.as_bytes());
|
||||
let signature = hex::encode(signer.finalize().into_bytes());
|
||||
let authorization = format!(
|
||||
"AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}",
|
||||
self.requester.key.id, scope, signed_headers, signature
|
||||
);
|
||||
all_headers.insert("authorization".to_owned(), authorization);
|
||||
|
||||
let mut request = Request::builder();
|
||||
for (k, v) in all_headers {
|
||||
request = request.header(k, v);
|
||||
}
|
||||
|
||||
let body = if let BodySignature::Streaming(size) = self.body_signature {
|
||||
to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope)
|
||||
} else {
|
||||
self.body.clone()
|
||||
};
|
||||
let request = request
|
||||
.uri(uri)
|
||||
.method(self.method.clone())
|
||||
.body(Body::from(body))
|
||||
.unwrap();
|
||||
self.requester.client.request(request).await
|
||||
}
|
||||
}
|
||||
|
||||
pub enum BodySignature {
|
||||
Unsigned,
|
||||
Classic,
|
||||
Streaming(usize),
|
||||
}
|
||||
|
||||
fn query_param_to_string(params: &HashMap<String, Option<String>>) -> String {
|
||||
if params.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
"?".to_owned()
|
||||
+ ¶ms
|
||||
.iter()
|
||||
.map(|(k, v)| {
|
||||
if let Some(v) = v {
|
||||
format!("{}={}", k, v)
|
||||
} else {
|
||||
k.clone()
|
||||
}
|
||||
})
|
||||
.collect::<Vec<String>>()
|
||||
.join("&")
|
||||
}
|
||||
|
||||
fn to_streaming_body(
|
||||
body: &[u8],
|
||||
chunk_size: usize,
|
||||
mut seed: String,
|
||||
hasher: Hmac<sha2::Sha256>,
|
||||
now: DateTime<Utc>,
|
||||
scope: &str,
|
||||
) -> Vec<u8> {
|
||||
const SHA_NULL: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
let now = now.format(signature::LONG_DATETIME).to_string();
|
||||
let mut res = Vec::with_capacity(body.len());
|
||||
for chunk in body.chunks(chunk_size).chain(std::iter::once(&[][..])) {
|
||||
let to_sign = format!(
|
||||
"AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}",
|
||||
now,
|
||||
scope,
|
||||
seed,
|
||||
SHA_NULL,
|
||||
hex::encode(garage_util::data::sha256sum(chunk))
|
||||
);
|
||||
|
||||
let mut hasher = hasher.clone();
|
||||
hasher.update(to_sign.as_bytes());
|
||||
seed = hex::encode(hasher.finalize().into_bytes());
|
||||
|
||||
let header = format!("{:x};chunk-signature={}\r\n", chunk.len(), seed);
|
||||
res.extend_from_slice(header.as_bytes());
|
||||
res.extend_from_slice(chunk);
|
||||
res.extend_from_slice(b"\r\n");
|
||||
}
|
||||
|
||||
res
|
||||
}
|
||||
@ -11,7 +11,7 @@ pub const DEFAULT_PORT: u16 = 49995;
|
||||
static GARAGE_TEST_SECRET: &str =
|
||||
"c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44";
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct Key {
|
||||
pub name: String,
|
||||
pub id: String,
|
||||
|
||||
@ -5,22 +5,31 @@ use ext::*;
|
||||
pub mod macros;
|
||||
|
||||
pub mod client;
|
||||
pub mod custom_requester;
|
||||
pub mod ext;
|
||||
pub mod garage;
|
||||
|
||||
use custom_requester::CustomRequester;
|
||||
|
||||
const REGION: Region = Region::from_static("garage-integ-test");
|
||||
|
||||
pub struct Context {
|
||||
pub garage: &'static garage::Instance,
|
||||
pub client: Client,
|
||||
pub custom_request: CustomRequester,
|
||||
}
|
||||
|
||||
impl Context {
|
||||
fn new() -> Self {
|
||||
let garage = garage::instance();
|
||||
let client = client::build_client(garage);
|
||||
let custom_request = CustomRequester::new(garage);
|
||||
|
||||
Context { garage, client }
|
||||
Context {
|
||||
garage,
|
||||
client,
|
||||
custom_request,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create an unique bucket with a random suffix.
|
||||
|
||||
@ -7,4 +7,5 @@ mod list;
|
||||
mod multipart;
|
||||
mod objects;
|
||||
mod simple;
|
||||
mod streaming_signature;
|
||||
mod website;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
use crate::common;
|
||||
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
|
||||
use aws_sdk_s3::ByteStream;
|
||||
use aws_sdk_s3::types::ByteStream;
|
||||
|
||||
const SZ_5MB: usize = 5 * 1024 * 1024;
|
||||
const SZ_10MB: usize = 10 * 1024 * 1024;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
use crate::common;
|
||||
use aws_sdk_s3::model::{Delete, ObjectIdentifier};
|
||||
use aws_sdk_s3::ByteStream;
|
||||
use aws_sdk_s3::types::ByteStream;
|
||||
|
||||
const STD_KEY: &str = "hello world";
|
||||
const CTRL_KEY: &str = "\x00\x01\x02\x00";
|
||||
|
||||
@ -2,7 +2,7 @@ use crate::common;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_simple() {
|
||||
use aws_sdk_s3::ByteStream;
|
||||
use aws_sdk_s3::types::ByteStream;
|
||||
|
||||
let ctx = common::context();
|
||||
let bucket = ctx.create_bucket("test-simple");
|
||||
|
||||
101
src/garage/tests/streaming_signature.rs
Normal file
101
src/garage/tests/streaming_signature.rs
Normal file
@ -0,0 +1,101 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::common;
|
||||
use common::custom_requester::BodySignature;
|
||||
use hyper::Method;
|
||||
|
||||
const STD_KEY: &str = "hello-world";
|
||||
//const CTRL_KEY: &str = "\x00\x01\x02\x00";
|
||||
const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_putobject_streaming() {
|
||||
let ctx = common::context();
|
||||
let bucket = ctx.create_bucket("putobject-streaming");
|
||||
|
||||
{
|
||||
// Send an empty object (can serve as a directory marker)
|
||||
// with a content type
|
||||
let etag = "\"d41d8cd98f00b204e9800998ecf8427e\"";
|
||||
let content_type = "text/csv";
|
||||
let mut headers = HashMap::new();
|
||||
headers.insert("content-type".to_owned(), content_type.to_owned());
|
||||
let _ = ctx
|
||||
.custom_request
|
||||
.builder(bucket.clone())
|
||||
.method(Method::PUT)
|
||||
.path(STD_KEY.to_owned())
|
||||
.unsigned_headers(headers)
|
||||
.vhost_style(true)
|
||||
.body(vec![])
|
||||
.body_signature(BodySignature::Streaming(10))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// assert_eq!(r.e_tag.unwrap().as_str(), etag);
|
||||
// We return a version ID here
|
||||
// We should check if Amazon is returning one when versioning is not enabled
|
||||
// assert!(r.version_id.is_some());
|
||||
|
||||
//let _version = r.version_id.unwrap();
|
||||
|
||||
let o = ctx
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(STD_KEY)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_bytes_eq!(o.body, b"");
|
||||
assert_eq!(o.e_tag.unwrap(), etag);
|
||||
// We do not return version ID
|
||||
// We should check if Amazon is returning one when versioning is not enabled
|
||||
// assert_eq!(o.version_id.unwrap(), _version);
|
||||
assert_eq!(o.content_type.unwrap(), content_type);
|
||||
assert!(o.last_modified.is_some());
|
||||
assert_eq!(o.content_length, 0);
|
||||
assert_eq!(o.parts_count, 0);
|
||||
assert_eq!(o.tag_count, 0);
|
||||
}
|
||||
|
||||
{
|
||||
let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
|
||||
|
||||
let _ = ctx
|
||||
.custom_request
|
||||
.builder(bucket.clone())
|
||||
.method(Method::PUT)
|
||||
//.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this
|
||||
//fail
|
||||
.path("abc".to_owned())
|
||||
.vhost_style(true)
|
||||
.body(BODY.to_vec())
|
||||
.body_signature(BodySignature::Streaming(16))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// assert_eq!(r.e_tag.unwrap().as_str(), etag);
|
||||
// assert!(r.version_id.is_some());
|
||||
|
||||
let o = ctx
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
//.key(CTRL_KEY)
|
||||
.key("abc")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_bytes_eq!(o.body, BODY);
|
||||
assert_eq!(o.e_tag.unwrap(), etag);
|
||||
assert!(o.last_modified.is_some());
|
||||
assert_eq!(o.content_length, 62);
|
||||
assert_eq!(o.parts_count, 0);
|
||||
assert_eq!(o.tag_count, 0);
|
||||
}
|
||||
}
|
||||
@ -2,7 +2,7 @@ use crate::common;
|
||||
use crate::common::ext::*;
|
||||
use aws_sdk_s3::{
|
||||
model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
|
||||
ByteStream,
|
||||
types::ByteStream,
|
||||
};
|
||||
use http::Request;
|
||||
use hyper::{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user