micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Validate content MD5 and SHA256 sums for PutObject and UploadPart

Browse Source
This commit is contained in:
Alex Auvolat 2020-07-15 15:31:13 +02:00
parent 6c7f9704ea
commit 1c70552f95
4 changed files with 104 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/api/api_server.rs
View File

@ -78,7 +78,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
))); )));
} }
let api_key = check_signature(&garage, &req).await?; let (api_key, content_sha256) = check_signature(&garage, &req).await?;
let allowed = match req.method() { let allowed = match req.method() {
&Method::HEAD | &Method::GET => api_key.allow_read(&bucket), &Method::HEAD | &Method::GET => api_key.allow_read(&bucket),
_ => api_key.allow_write(&bucket), _ => api_key.allow_write(&bucket),
@ -114,7 +114,16 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
// UploadPart query // UploadPart query
let part_number = params.get("partnumber").unwrap(); let part_number = params.get("partnumber").unwrap();
let upload_id = params.get("uploadid").unwrap(); let upload_id = params.get("uploadid").unwrap();
Ok(handle_put_part(garage, req, &bucket, &key, part_number, upload_id).await?) Ok(handle_put_part(
garage,
req,
&bucket,
&key,
part_number,
upload_id,
content_sha256,
)
.await?)
} else if req.headers().contains_key("x-amz-copy-source") { } else if req.headers().contains_key("x-amz-copy-source") {
// CopyObject query // CopyObject query
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?; let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
@ -134,7 +143,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
Ok(handle_copy(garage, &bucket, &key, &source_bucket, &source_key).await?) Ok(handle_copy(garage, &bucket, &key, &source_bucket, &source_key).await?)
} else { } else {
// PutObject query // PutObject query
Ok(handle_put(garage, req, &bucket, &key).await?) Ok(handle_put(garage, req, &bucket, &key, content_sha256).await?)
} }
} }
&Method::DELETE => { &Method::DELETE => {

8
src/api/s3_get.rs
View File

@ -29,11 +29,11 @@ fn object_headers(
.header("Last-Modified", date_str) .header("Last-Modified", date_str)
.header("Accept-Ranges", format!("bytes")); .header("Accept-Ranges", format!("bytes"));
for (k, v) in version_meta.headers.other.iter() { for (k, v) in version_meta.headers.other.iter() {
resp = resp.header(k, v.to_string()); resp = resp.header(k, v.to_string());
} }
resp resp
} }
pub async fn handle_head( pub async fn handle_head(

78
src/api/s3_put.rs
View File

@ -2,9 +2,10 @@ use std::collections::{BTreeMap, VecDeque};
use std::fmt::Write; use std::fmt::Write;
use std::sync::Arc; use std::sync::Arc;
use md5::{Md5, Digest};
use futures::stream::*; use futures::stream::*;
use hyper::{Body, Request, Response}; use hyper::{Body, Request, Response};
use md5::{Digest as Md5Digest, Md5};
use sha2::{Digest as Sha256Digest, Sha256};
use garage_table::*; use garage_table::*;
use garage_util::data::*; use garage_util::data::*;
@ -23,9 +24,14 @@ pub async fn handle_put(
req: Request<Body>, req: Request<Body>,
bucket: &str, bucket: &str,
key: &str, key: &str,
content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let version_uuid = gen_uuid(); let version_uuid = gen_uuid();
let headers = get_headers(&req)?; let headers = get_headers(&req)?;
let content_md5 = match req.headers().get("content-md5") {
Some(x) => Some(x.to_str()?.to_string()),
None => None,
};
let body = req.into_body(); let body = req.into_body();
@ -42,9 +48,9 @@ pub async fn handle_put(
}; };
if first_block.len() < INLINE_THRESHOLD { if first_block.len() < INLINE_THRESHOLD {
let mut md5sum = Md5::new(); let mut md5sum = Md5::new();
md5sum.update(&first_block[..]); md5sum.update(&first_block[..]);
let etag = hex::encode(md5sum.finalize()); let etag = hex::encode(md5sum.finalize());
object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline( object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline(
ObjectVersionMeta { ObjectVersionMeta {
@ -66,7 +72,7 @@ pub async fn handle_put(
let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]); let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]);
garage.object_table.insert(&object).await?; garage.object_table.insert(&object).await?;
let (total_size, etag) = read_and_put_blocks( let (total_size, md5sum, sha256sum) = read_and_put_blocks(
&garage, &garage,
version, version,
1, 1,
@ -76,13 +82,26 @@ pub async fn handle_put(
) )
.await?; .await?;
if let Some(expected_sha256) = content_sha256 {
if expected_sha256 != sha256sum {
return Err(Error::Message(format!(
"Unable to validate x-amz-content-sha256"
)));
}
}
if let Some(expected_md5) = content_md5 {
if expected_md5.trim_matches('"') != md5sum {
return Err(Error::Message(format!("Unable to validate content-md5")));
}
}
// TODO: if at any step we have an error, we should undo everything we did // TODO: if at any step we have an error, we should undo everything we did
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock( object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
ObjectVersionMeta { ObjectVersionMeta {
headers, headers,
size: total_size, size: total_size,
etag: etag.clone(), etag: md5sum.clone(),
}, },
first_block_hash, first_block_hash,
)); ));
@ -90,7 +109,7 @@ pub async fn handle_put(
let object = Object::new(bucket.into(), key.into(), vec![object_version]); let object = Object::new(bucket.into(), key.into(), vec![object_version]);
garage.object_table.insert(&object).await?; garage.object_table.insert(&object).await?;
Ok(put_response(version_uuid, etag)) Ok(put_response(version_uuid, md5sum))
} }
async fn read_and_put_blocks( async fn read_and_put_blocks(
@ -100,9 +119,11 @@ async fn read_and_put_blocks(
first_block: Vec<u8>, first_block: Vec<u8>,
first_block_hash: Hash, first_block_hash: Hash,
chunker: &mut BodyChunker, chunker: &mut BodyChunker,
) -> Result<(u64, String), Error> { ) -> Result<(u64, String, Hash), Error> {
let mut md5sum = Md5::new(); let mut md5sum = Md5::new();
md5sum.update(&first_block[..]); let mut sha256sum = Sha256::new();
md5sum.update(&first_block[..]);
sha256sum.input(&first_block[..]);
let mut next_offset = first_block.len(); let mut next_offset = first_block.len();
let mut put_curr_version_block = put_block_meta( let mut put_curr_version_block = put_block_meta(
@ -121,7 +142,8 @@ async fn read_and_put_blocks(
let (_, _, next_block) = let (_, _, next_block) =
futures::try_join!(put_curr_block, put_curr_version_block, chunker.next())?; futures::try_join!(put_curr_block, put_curr_version_block, chunker.next())?;
if let Some(block) = next_block { if let Some(block) = next_block {
md5sum.update(&block[..]); md5sum.update(&block[..]);
sha256sum.input(&block[..]);
let block_hash = hash(&block[..]); let block_hash = hash(&block[..]);
let block_len = block.len(); let block_len = block.len();
put_curr_version_block = put_block_meta( put_curr_version_block = put_block_meta(
@ -139,9 +161,15 @@ async fn read_and_put_blocks(
} }
} }
let total_size = next_offset as u64; let total_size = next_offset as u64;
let md5sum = hex::encode(md5sum.finalize()); let md5sum = hex::encode(md5sum.finalize());
Ok((total_size, md5sum))
let sha256sum = sha256sum.result();
let mut hash = [0u8; 32];
hash.copy_from_slice(&sha256sum[..]);
let sha256sum = Hash::from(hash);
Ok((total_size, md5sum, sha256sum))
} }
async fn put_block_meta( async fn put_block_meta(
@ -267,6 +295,7 @@ pub async fn handle_put_part(
key: &str, key: &str,
part_number_str: &str, part_number_str: &str,
upload_id: &str, upload_id: &str,
content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
// Check parameters // Check parameters
let part_number = part_number_str let part_number = part_number_str
@ -276,6 +305,11 @@ pub async fn handle_put_part(
let version_uuid = let version_uuid =
uuid_from_str(upload_id).map_err(|_| Error::BadRequest(format!("Invalid upload ID")))?; uuid_from_str(upload_id).map_err(|_| Error::BadRequest(format!("Invalid upload ID")))?;
let content_md5 = match req.headers().get("content-md5") {
Some(x) => Some(x.to_str()?.to_string()),
None => None,
};
// Read first chuck, and at the same time try to get object to see if it exists // Read first chuck, and at the same time try to get object to see if it exists
let mut chunker = BodyChunker::new(req.into_body(), garage.config.block_size); let mut chunker = BodyChunker::new(req.into_body(), garage.config.block_size);
@ -307,7 +341,7 @@ pub async fn handle_put_part(
// Copy block to store // Copy block to store
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]); let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
let first_block_hash = hash(&first_block[..]); let first_block_hash = hash(&first_block[..]);
read_and_put_blocks( let (_, md5sum, sha256sum) = read_and_put_blocks(
&garage, &garage,
version, version,
part_number, part_number,
@ -317,6 +351,20 @@ pub async fn handle_put_part(
) )
.await?; .await?;
if let Some(expected_md5) = content_md5 {
if expected_md5.trim_matches('"') != md5sum {
return Err(Error::Message(format!("Unable to validate content-md5")));
}
}
if let Some(expected_sha256) = content_sha256 {
if expected_sha256 != sha256sum {
return Err(Error::Message(format!(
"Unable to validate x-amz-content-sha256"
)));
}
}
Ok(Response::new(Body::from(vec![]))) Ok(Response::new(Body::from(vec![])))
} }

28
src/api/signature.rs
View File

@ -6,6 +6,7 @@ use hyper::{Body, Method, Request};
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use garage_table::*; use garage_table::*;
use garage_util::data::Hash;
use garage_util::error::Error; use garage_util::error::Error;
use garage_model::garage::Garage; use garage_model::garage::Garage;
@ -18,7 +19,10 @@ const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
type HmacSha256 = Hmac<Sha256>; type HmacSha256 = Hmac<Sha256>;
pub async fn check_signature(garage: &Garage, request: &Request<Body>) -> Result<Key, Error> { pub async fn check_signature(
garage: &Garage,
request: &Request<Body>,
) -> Result<(Key, Option<Hash>), Error> {
let mut headers = HashMap::new(); let mut headers = HashMap::new();
for (key, val) in request.headers() { for (key, val) in request.headers() {
headers.insert(key.to_string(), val.to_str()?.to_string()); headers.insert(key.to_string(), val.to_str()?.to_string());
@ -97,7 +101,21 @@ pub async fn check_signature(garage: &Garage, request: &Request<Body>) -> Result
return Err(Error::Forbidden(format!("Invalid signature"))); return Err(Error::Forbidden(format!("Invalid signature")));
} }
Ok(key) let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" {
None
} else {
let bytes = hex::decode(authorization.content_sha256).or(Err(Error::BadRequest(
format!("Invalid content sha256 hash"),
)))?;
let mut hash = [0u8; 32];
if bytes.len() != 32 {
return Err(Error::BadRequest(format!("Invalid content sha256 hash")));
}
hash.copy_from_slice(&bytes[..]);
Some(Hash::from(hash))
};
Ok((key, content_sha256))
} }
struct Authorization { struct Authorization {
@ -193,13 +211,17 @@ fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Author
.ok_or(Error::BadRequest(format!( .ok_or(Error::BadRequest(format!(
"X-Amz-Signature not found in query parameters" "X-Amz-Signature not found in query parameters"
)))?; )))?;
let content_sha256 = headers
.get("x-amz-content-sha256")
.map(|x| x.as_str())
.unwrap_or("UNSIGNED-PAYLOAD");
Ok(Authorization { Ok(Authorization {
key_id, key_id,
scope, scope,
signed_headers: signed_headers.to_string(), signed_headers: signed_headers.to_string(),
signature: signature.to_string(), signature: signature.to_string(),
content_sha256: "UNSIGNED-PAYLOAD".to_string(), content_sha256: content_sha256.to_string(),
}) })
} }