Implement sending blocks to nodes that need them

This commit is contained in:
Alex Auvolat 2020-04-17 19:16:08 +02:00
parent db1c4222ce
commit 4abfb75509
5 changed files with 69 additions and 6 deletions

View File

@ -1,3 +1,3 @@
all: all:
cargo fmt cargo fmt || true
cargo build cargo build

5
TODO
View File

@ -1,9 +1,7 @@
Replication Replication
----------- -----------
- for each interval of tokens, we know the list of nodes that are responsible Finish the thing that sends blocks to other nodes if needed before deleting them locally.
- every node watches the current ring and state of the network
- and thus determines the interval of tokens for which they are responsible
How are we going to test that our replication method works correctly? How are we going to test that our replication method works correctly?
We will have to introduce lots of dummy data and then add/remove nodes many times. We will have to introduce lots of dummy data and then add/remove nodes many times.
@ -12,7 +10,6 @@ We will have to introduce lots of dummy data and then add/remove nodes many time
To do list To do list
---------- ----------
- important: check block values on read and repare corrupted block contents
- less a priority: hinted handoff - less a priority: hinted handoff
- FIXME in rpc_server when garage shuts down and futures can be interrupted - FIXME in rpc_server when garage shuts down and futures can be interrupted
(tokio::spawn should be replaced by a new function background::spawn_joinable) (tokio::spawn should be replaced by a new function background::spawn_joinable)

View File

@ -3,6 +3,7 @@ use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use arc_swap::ArcSwapOption; use arc_swap::ArcSwapOption;
use futures::future::*;
use futures::stream::*; use futures::stream::*;
use tokio::fs; use tokio::fs;
use tokio::prelude::*; use tokio::prelude::*;
@ -16,6 +17,8 @@ use crate::proto::*;
use crate::rpc_client::*; use crate::rpc_client::*;
use crate::server::Garage; use crate::server::Garage;
const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
pub struct BlockManager { pub struct BlockManager {
pub data_dir: PathBuf, pub data_dir: PathBuf,
pub rc: sled::Tree, pub rc: sled::Tree,
@ -102,6 +105,22 @@ impl BlockManager {
})) }))
} }
pub async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
let needed = self
.rc
.get(hash.as_ref())?
.map(|x| u64_from_bytes(x.as_ref()) > 0)
.unwrap_or(false);
if needed {
let mut path = self.data_dir.clone();
path.push(hex::encode(hash.as_ref()));
let exists = fs::metadata(&path).await.is_ok();
Ok(!exists)
} else {
Ok(false)
}
}
fn block_dir(&self, hash: &Hash) -> PathBuf { fn block_dir(&self, hash: &Hash) -> PathBuf {
let mut path = self.data_dir.clone(); let mut path = self.data_dir.clone();
path.push(hex::encode(&hash.as_slice()[0..1])); path.push(hex::encode(&hash.as_slice()[0..1]));
@ -191,7 +210,47 @@ impl BlockManager {
.await?; .await?;
let needed_by_others = !active_refs.is_empty(); let needed_by_others = !active_refs.is_empty();
if needed_by_others { if needed_by_others {
// TODO check they have it and send it if not let ring = garage.system.ring.borrow().clone();
let who = ring.walk_ring(&hash, garage.system.config.data_replication_factor);
let msg = Message::NeedBlockQuery(hash.clone());
let who_needs_fut = who
.iter()
.map(|to| rpc_call(garage.system.clone(), to, &msg, NEED_BLOCK_QUERY_TIMEOUT));
let who_needs = join_all(who_needs_fut).await;
let mut need_nodes = vec![];
let mut errors = 0;
for (node, needed) in who.into_iter().zip(who_needs.iter()) {
match needed {
Ok(Message::NeedBlockReply(true)) => {
need_nodes.push(node);
}
Err(_) => {
errors += 1;
}
_ => (),
}
}
if errors > (garage.system.config.data_replication_factor - 1) / 2 {
return Err(Error::Message(format!(
"Should delete block, but not enough nodes confirm that they have it."
)));
}
if need_nodes.len() > 0 {
let put_block_message = self.read_block(hash).await?;
for resp in rpc_call_many(
garage.system.clone(),
&need_nodes[..],
put_block_message,
BLOCK_RW_TIMEOUT,
)
.await
{
resp?;
}
}
} }
fs::remove_file(path).await?; fs::remove_file(path).await?;
self.resync_queue.remove(&hash)?; self.resync_queue.remove(&hash)?;

View File

@ -20,6 +20,8 @@ pub enum Message {
GetBlock(Hash), GetBlock(Hash),
PutBlock(PutBlockMessage), PutBlock(PutBlockMessage),
NeedBlockQuery(Hash),
NeedBlockReply(bool),
TableRPC(String, #[serde(with = "serde_bytes")] Vec<u8>), TableRPC(String, #[serde(with = "serde_bytes")] Vec<u8>),
} }

View File

@ -66,6 +66,11 @@ async fn handler(
tokio::spawn(write_fut).await? tokio::spawn(write_fut).await?
} }
Message::GetBlock(h) => garage.block_manager.read_block(&h).await, Message::GetBlock(h) => garage.block_manager.read_block(&h).await,
Message::NeedBlockQuery(h) => garage
.block_manager
.need_block(&h)
.await
.map(Message::NeedBlockReply),
Message::TableRPC(table, msg) => { Message::TableRPC(table, msg) => {
// Same trick for table RPCs than for PutBlock // Same trick for table RPCs than for PutBlock