From acd49de9f97bd27409232691262bd5827983388d Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 15 Nov 2023 13:07:42 +0100
Subject: [PATCH] rpc: fix write set quorums

---
 src/api/common_error.rs |  8 ++------
 src/model/k2v/rpc.rs    |  2 +-
 src/rpc/rpc_helper.rs   | 18 +++++++++++++++---
 src/util/error.rs       |  7 ++++---
 4 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/src/api/common_error.rs b/src/api/common_error.rs
index 20f9f266..ecb22fd8 100644
--- a/src/api/common_error.rs
+++ b/src/api/common_error.rs
@@ -53,9 +53,7 @@ impl CommonError {
 	pub fn http_status_code(&self) -> StatusCode {
 		match self {
 			CommonError::InternalError(
-				GarageError::Timeout
-				| GarageError::RemoteError(_)
-				| GarageError::Quorum(_, _, _, _),
+				GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..),
 			) => StatusCode::SERVICE_UNAVAILABLE,
 			CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => {
 				StatusCode::INTERNAL_SERVER_ERROR
@@ -72,9 +70,7 @@ impl CommonError {
 		match self {
 			CommonError::Forbidden(_) => "AccessDenied",
 			CommonError::InternalError(
-				GarageError::Timeout
-				| GarageError::RemoteError(_)
-				| GarageError::Quorum(_, _, _, _),
+				GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..),
 			) => "ServiceUnavailable",
 			CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => {
 				"InternalError"
diff --git a/src/model/k2v/rpc.rs b/src/model/k2v/rpc.rs
index 863a068a..3c759181 100644
--- a/src/model/k2v/rpc.rs
+++ b/src/model/k2v/rpc.rs
@@ -344,7 +344,7 @@ impl K2VRpcHandler {
 		}
 		if errors.len() > nodes.len() - quorum {
 			let errors = errors.iter().map(|e| format!("{}", e)).collect::<Vec<_>>();
-			return Err(Error::Quorum(quorum, resps.len(), nodes.len(), errors).into());
+			return Err(Error::Quorum(quorum, None, resps.len(), nodes.len(), errors).into());
 		}
 
 		// Take all returned items into account to produce the response.
diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs
index 12d073b6..1bad495b 100644
--- a/src/rpc/rpc_helper.rs
+++ b/src/rpc/rpc_helper.rs
@@ -327,7 +327,13 @@ impl RpcHelper {
 			Ok(successes)
 		} else {
 			let errors = errors.iter().map(|e| format!("{}", e)).collect::<Vec<_>>();
-			Err(Error::Quorum(quorum, successes.len(), to.len(), errors))
+			Err(Error::Quorum(
+				quorum,
+				None,
+				successes.len(),
+				to.len(),
+				errors,
+			))
 		}
 	}
 
@@ -469,7 +475,7 @@ impl RpcHelper {
 				}
 			}
 
-			if set_counters.iter().all(|x| x.0 > quorum) {
+			if set_counters.iter().all(|x| x.0 >= quorum) {
 				// Success
 
 				// Continue all other requets in background
@@ -492,6 +498,12 @@ impl RpcHelper {
 
 		// Failure, could not get quorum
 		let errors = errors.iter().map(|e| format!("{}", e)).collect::<Vec<_>>();
-		Err(Error::Quorum(quorum, successes.len(), peers.len(), errors))
+		Err(Error::Quorum(
+			quorum,
+			Some(to_sets.len()),
+			successes.len(),
+			peers.len(),
+			errors,
+		))
 	}
 }
diff --git a/src/util/error.rs b/src/util/error.rs
index 3fcee71d..be7cdfdb 100644
--- a/src/util/error.rs
+++ b/src/util/error.rs
@@ -55,13 +55,14 @@ pub enum Error {
 	Timeout,
 
 	#[error(
-		display = "Could not reach quorum of {}. {} of {} request succeeded, others returned errors: {:?}",
+		display = "Could not reach quorum of {} (sets={:?}). {} of {} request succeeded, others returned errors: {:?}",
 		_0,
 		_1,
 		_2,
-		_3
+		_3,
+		_4
 	)]
-	Quorum(usize, usize, usize, Vec<String>),
+	Quorum(usize, Option<usize>, usize, usize, Vec<String>),
 
 	#[error(display = "Unexpected RPC message: {}", _0)]
 	UnexpectedRpcMessage(String),