From 80c67e6b663a42a2fea866cd677d7088a25adfd3 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 7 Feb 2015 00:21:34 +0100 Subject: New entries per node should be a set. Chunk sendlog. --- tools/merge.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py index 974bab0..f4a007d 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -169,8 +169,8 @@ entries_to_fetch = {} for storagenode in storagenodes: print "getting new entries from", storagenode - new_entries_per_node[storagenode] = get_new_entries(storagenode) - new_entries.update(set(new_entries_per_node[storagenode])) + new_entries_per_node[storagenode] = set(get_new_entries(storagenode)) + new_entries.update(new_entries_per_node[storagenode]) entries_to_fetch[storagenode] = [] timing_point(timing, "get new entries") @@ -191,7 +191,8 @@ for hash in new_entries: added_entries = 0 for storagenode in storagenodes: - for chunk in chunks(entries_to_fetch[storagenode], 10): + print "getting", len(entries_to_fetch[storagenode]), "entries from", storagenode + for chunk in chunks(entries_to_fetch[storagenode], 100): entries = get_entries(storagenode, chunk) for hash in chunk: entry = entries[hash] @@ -231,7 +232,11 @@ for frontendnode in frontendnodes: timing_point(timing, "get curpos") print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] - sendlog(frontendnode, {"start": curpos, "hashes": entries}) + for chunk in chunks(entries, 1000): + sendlog(frontendnode, {"start": curpos, "hashes": chunk}) + curpos += len(chunk) + print curpos, + sys.stdout.flush() timing_point(timing, "sendlog") print "log sent" missingentries = get_missingentries(frontendnode) -- cgit v1.1