micropelago/garage
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Evaluate hash functions

Browse Source
This commit is contained in:
Alex Auvolat 2021-02-21 15:11:15 +01:00
parent 217b0dfd68
commit e59322041a
1 changed files with 25 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
script/simulate_ring.py
View File

@ -2,6 +2,13 @@ import hashlib
import bisect import bisect
import xxhash import xxhash
def hash_str(s):
xxh = xxhash.xxh64()
xxh.update(s.encode('ascii'))
return xxh.hexdigest()
def sha256_str(s):
return hashlib.sha256(s.encode('ascii')).hexdigest()
def walk_ring_from_pos(tokens, dcs, start, rep): def walk_ring_from_pos(tokens, dcs, start, rep):
ret = [] ret = []
@ -29,22 +36,24 @@ def count_tokens_per_node(tokens):
for node, ntok in sorted(list(tokens_of_node.items())): for node, ntok in sorted(list(tokens_of_node.items())):
print(node, ": ", ntok) print(node, ": ", ntok)
def method1(nodes): def method1(nodes):
tokens = [] tokens = []
dcs = set() dcs = set()
for (node_id, dc, n_tokens) in nodes: for (node_id, dc, n_tokens) in nodes:
dcs |= set([dc]) dcs |= set([dc])
for i in range(n_tokens): for i in range(n_tokens):
token = hashlib.sha256(f"{node_id} {i}".encode('ascii')).hexdigest() token = hash_str(f"{node_id} {i}")
tokens.append((token, dc, node_id)) tokens.append((token, dc, node_id))
tokens.sort(key=lambda tok: tok[0]) tokens.sort(key=lambda tok: tok[0])
print(tokens)
count_tokens_per_node(tokens) count_tokens_per_node(tokens)
space_of_node = {} space_of_node = {}
def walk_ring(v, rep): def walk_ring(v, rep):
i = bisect.bisect_left([tok for tok, _, _ in tokens], hashlib.sha256(v).hexdigest()) i = bisect.bisect_left([tok for tok, _, _ in tokens], hash_str(v))
return walk_ring_from_pos(tokens, dcs, i, rep) return walk_ring_from_pos(tokens, dcs, i, rep)
return walk_ring return walk_ring
@ -57,10 +66,7 @@ def method2(nodes):
h, hn, hndc = None, None, None h, hn, hndc = None, None, None
for (node_id, node_dc, n_tokens) in nodes: for (node_id, node_dc, n_tokens) in nodes:
for tok in range(n_tokens): for tok in range(n_tokens):
#hnode = hashlib.sha256(f"{i} {node_id} {tok}".encode('ascii')).hexdigest() hnode = hash_str(f"partition {i} node {node_id} token {tok}")
xxh = xxhash.xxh64()
xxh.update(f"partition {i} node {node_id} token {tok}".encode('ascii'))
hnode = xxh.digest()
if h is None or hnode < h: if h is None or hnode < h:
h = hnode h = hnode
hn = node_id hn = node_id
@ -74,8 +80,10 @@ def method2(nodes):
def walk_ring(v, rep): def walk_ring(v, rep):
vh = hashlib.sha256(v).digest() xxh = xxhash.xxh32()
i = (vh[0]<<8 | vh[1]) % (2**partition_bits) xxh.update(v.encode('ascii'))
vh = xxh.intdigest()
i = vh % (2**partition_bits)
return walk_ring_from_pos(partition_nodes, dcs, i, rep) return walk_ring_from_pos(partition_nodes, dcs, i, rep)
return walk_ring return walk_ring
@ -85,7 +93,7 @@ def method2(nodes):
def evaluate_method(walk_ring): def evaluate_method(walk_ring):
node_data_counts = {} node_data_counts = {}
for i in range(100000): for i in range(100000):
nodes = walk_ring(f"{i}".encode('ascii'), 3) nodes = walk_ring(f"{i}", 3)
for n in nodes: for n in nodes:
if n not in node_data_counts: if n not in node_data_counts:
node_data_counts[n] = 0 node_data_counts[n] = 0
@ -96,25 +104,20 @@ def evaluate_method(walk_ring):
if __name__ == "__main__": if __name__ == "__main__":
nodes = [('digitale', 'atuin', 10),
('drosera', 'atuin', 10),
('datura', 'atuin', 10),
('io', 'jupiter', 20),
('meta', 'pipo', 10),
('mega', 'pipo', 10),
('mina', 'pipo', 10),
#('moni', 'pipo', 10),
#('mimi', 'pipo', 10),
#('mesi', 'pipo', 10),
]
print("------") print("------")
print("method 1") print("method 1")
nodes = [('digitale', 'atuin', 64),
('drosera', 'atuin', 64),
('datura', 'atuin', 64),
('io', 'jupiter', 128)]
method1_walk_ring = method1(nodes) method1_walk_ring = method1(nodes)
evaluate_method(method1_walk_ring) evaluate_method(method1_walk_ring)
print("------") print("------")
print("method 2") print("method 2")
nodes = [('digitale', 'atuin', 10),
('drosera', 'atuin', 10),
('datura', 'atuin', 10),
('io', 'jupiter', 20)]
method2_walk_ring = method2(nodes) method2_walk_ring = method2(nodes)
evaluate_method(method2_walk_ring) evaluate_method(method2_walk_ring)