micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
2832be4396
garage/src/rpc_server.rs

192 lines
5.6 KiB
Rust
Raw Normal View History

Some more basic work
2020-04-06 17:55:39 +00:00
use std::net::SocketAddr;
use std::sync::Arc;
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
use bytes::IntoBuf;
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
use futures::future::Future;
We don't want the debugging to fail, actually.
2020-04-10 21:43:35 +00:00
use futures_util::future::*;
Trying to do TLS
2020-04-12 13:51:19 +00:00
use futures_util::stream::*;
Some more basic work
2020-04-06 17:55:39 +00:00
use hyper::server::conn::AddrStream;
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
use hyper::service::{make_service_fn, service_fn};
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
use hyper::{Body, Method, Request, Response, Server, StatusCode};
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
use serde::Serialize;
Trying to do TLS
2020-04-12 13:51:19 +00:00
use tokio::net::{TcpListener, TcpStream};
use tokio_rustls::server::TlsStream;
use tokio_rustls::TlsAcceptor;
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
Something works
2020-04-09 16:43:53 +00:00
use crate::data::rmp_to_vec_all_named;
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
use crate::error::Error;
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
use crate::proto::Message;
Some work in actually storing things
2020-04-08 20:00:41 +00:00
use crate::server::Garage;
Trying to do TLS
2020-04-12 13:51:19 +00:00
use crate::tls_util;
Why is it not Sync??
2020-04-09 21:45:07 +00:00
We don't want the debugging to fail, actually.
2020-04-10 21:43:35 +00:00
fn debug_serialize<T: Serialize>(x: T) -> String {
match serde_json::to_string(&x) {
Ok(ss) => {
if ss.len() > 100 {
ss[..100].to_string()
} else {
ss
}
}
Err(e) => format!("<JSON serialization error: {}>", e),
Why is it not Sync??
2020-04-09 21:45:07 +00:00
}
}
Some more basic work
2020-04-06 17:55:39 +00:00
fn err_to_msg(x: Result<Message, Error>) -> Message {
match x {
Err(e) => Message::Error(format!("{}", e)),
Ok(msg) => msg,
}
}
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
async fn handler(
garage: Arc<Garage>,
req: Request<Body>,
addr: SocketAddr,
) -> Result<Response<Body>, Error> {
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
if req.method() != &Method::POST {
let mut bad_request = Response::default();
*bad_request.status_mut() = StatusCode::BAD_REQUEST;
return Ok(bad_request);
}
let whole_body = hyper::body::to_bytes(req.into_body()).await?;
Some more basic work
2020-04-06 17:55:39 +00:00
let msg = rmp_serde::decode::from_read::<_, Message>(whole_body.into_buf())?;
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
eprintln!(
"RPC from {}: {} ({} bytes)",
addr,
We don't want the debugging to fail, actually.
2020-04-10 21:43:35 +00:00
debug_serialize(&msg),
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
whole_body.len()
);
Some more basic work
2020-04-06 17:55:39 +00:00
Some work in actually storing things
2020-04-08 20:00:41 +00:00
let sys = garage.system.clone();
fixes - make block_put call uninterruptible by client - used meta_replication_factor instead of data_replication_factor - listen on ipv6
2020-04-12 10:10:33 +00:00
let resp = err_to_msg(match msg {
Message::Ping(ping) => sys.handle_ping(&addr, &ping).await,
We don't want the debugging to fail, actually.
2020-04-10 21:43:35 +00:00
Keep network status & ring in a tokio::sync::watch advantages - reads don't prevent preparing writes - can be followed from other parts of the system by cloning the receiver
2020-04-11 21:53:32 +00:00
Message::PullStatus => sys.handle_pull_status(),
Message::PullConfig => sys.handle_pull_config(),
fixes - make block_put call uninterruptible by client - used meta_replication_factor instead of data_replication_factor - listen on ipv6
2020-04-12 10:10:33 +00:00
Message::AdvertiseNodesUp(adv) => sys.handle_advertise_nodes_up(&adv).await,
Message::AdvertiseConfig(adv) => sys.handle_advertise_config(&adv).await,
Message::PutBlock(m) => {
// A RPC can be interrupted in the middle, however we don't want to write partial blocks,
// which might happen if the write_block() future is cancelled in the middle.
// To solve this, the write itself is in a spawned task that has its own separate lifetime,
// and the request handler simply sits there waiting for the task to finish.
// (if it's cancelled, that's not an issue)
// (TODO FIXME except if garage happens to shut down at that point)
Begin work on sync...
2020-04-16 12:50:49 +00:00
let write_fut = async move { garage.block_manager.write_block(&m.hash, &m.data).await };
fixes - make block_put call uninterruptible by client - used meta_replication_factor instead of data_replication_factor - listen on ipv6
2020-04-12 10:10:33 +00:00
tokio::spawn(write_fut).await?
}
Local refcounting of blocks
2020-04-11 21:00:26 +00:00
Message::GetBlock(h) => garage.block_manager.read_block(&h).await,
We don't want the debugging to fail, actually.
2020-04-10 21:43:35 +00:00
Some work in actually storing things
2020-04-08 20:00:41 +00:00
Message::TableRPC(table, msg) => {
Fix table RPC to not be interruptible
2020-04-12 21:05:53 +00:00
// Same trick for table RPCs than for PutBlock
let op_fut = async move {
if let Some(rpc_handler) = garage.table_rpc_handlers.get(&table) {
rpc_handler
.handle(&msg[..])
.await
.map(|rep| Message::TableRPC(table.to_string(), rep))
} else {
Ok(Message::Error(format!("Unknown table: {}", table)))
}
};
tokio::spawn(op_fut).await?
Some work in actually storing things
2020-04-08 20:00:41 +00:00
}
Ununderstandable error
2020-04-06 20:27:51 +00:00
Some more basic work
2020-04-06 17:55:39 +00:00
_ => Ok(Message::Error(format!("Unexpected message: {:?}", msg))),
});
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
We don't want the debugging to fail, actually.
2020-04-10 21:43:35 +00:00
eprintln!("reply to {}: {}", addr, debug_serialize(&resp));
Something works
2020-04-09 16:43:53 +00:00
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
Ok(Response::new(Body::from(rmp_to_vec_all_named(&resp)?)))
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
}
Fix the Sync issue. Details: So the HTTP client future of Hyper is not Sync, thus the stream that read blocks wasn't either. However Hyper's default Body type requires a stream to be Sync for wrap_stream. Solution: reimplement a custom HTTP body type.
2020-04-10 20:01:48 +00:00
pub async fn run_rpc_server(
garage: Arc<Garage>,
shutdown_signal: impl Future<Output = ()>,
Background task runner that replaces tokio::spawn
2020-04-11 16:51:11 +00:00
) -> Result<(), Error> {
fixes - make block_put call uninterruptible by client - used meta_replication_factor instead of data_replication_factor - listen on ipv6
2020-04-12 10:10:33 +00:00
let bind_addr = ([0, 0, 0, 0, 0, 0, 0, 0], garage.system.config.rpc_port).into();
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
Trying to do TLS
2020-04-12 13:51:19 +00:00
if let Some(tls_config) = &garage.system.config.rpc_tls {
let ca_certs = tls_util::load_certs(&tls_config.ca_cert)?;
let node_certs = tls_util::load_certs(&tls_config.node_cert)?;
let node_key = tls_util::load_private_key(&tls_config.node_key)?;
let mut ca_store = rustls::RootCertStore::empty();
for crt in ca_certs.iter() {
ca_store.add(crt)?;
Some more basic work
2020-04-06 17:55:39 +00:00
}
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
Trying to do TLS
2020-04-12 13:51:19 +00:00
let mut config =
rustls::ServerConfig::new(rustls::AllowAnyAuthenticatedClient::new(ca_store));
TLS works \o/ So, the issues were: - webpki does not support IP addresses as DNS names in URLs, so I hacked the HttpsConnector to always provide a fixed string as the DNS name for server certificate validation - the certificate requied a SAN section which was complicated to build but eventually the solution is there in genkeys.sh
2020-04-12 17:00:30 +00:00
config.set_single_cert([&node_certs[..], &ca_certs[..]].concat(), node_key)?;
Trying to do TLS
2020-04-12 13:51:19 +00:00
let tls_acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
let mut listener = TcpListener::bind(&bind_addr).await?;
let incoming = listener.incoming().filter_map(|socket| async {
match socket {
Ok(stream) => match tls_acceptor.clone().accept(stream).await {
Ok(x) => Some(Ok::<_, hyper::Error>(x)),
Err(e) => {
eprintln!("RPC server TLS error: {}", e);
None
}
},
Err(_) => None,
}
});
let incoming = hyper::server::accept::from_stream(incoming);
let service = make_service_fn(|conn: &TlsStream<TcpStream>| {
let client_addr = conn
.get_ref()
.0
.peer_addr()
.unwrap_or(([0, 0, 0, 0], 0).into());
let garage = garage.clone();
async move {
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
let garage = garage.clone();
handler(garage, req, client_addr).map_err(|e| {
eprintln!("RPC handler error: {}", e);
e
})
}))
}
});
let server = Server::builder(incoming).serve(service);
let graceful = server.with_graceful_shutdown(shutdown_signal);
println!("RPC server listening on http://{}", bind_addr);
graceful.await?;
} else {
let service = make_service_fn(|conn: &AddrStream| {
let client_addr = conn.remote_addr();
let garage = garage.clone();
async move {
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
let garage = garage.clone();
handler(garage, req, client_addr).map_err(|e| {
eprintln!("RPC handler error: {}", e);
e
})
}))
}
});
let server = Server::bind(&bind_addr).serve(service);
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
Trying to do TLS
2020-04-12 13:51:19 +00:00
let graceful = server.with_graceful_shutdown(shutdown_signal);
println!("RPC server listening on http://{}", bind_addr);
graceful.await?;
}
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
Background task runner that replaces tokio::spawn
2020-04-11 16:51:11 +00:00
Ok(())
First commit: skeleton for something great
2020-04-05 21:33:42 +00:00
}
Reference in New Issue Copy Permalink