diff options
| -rwxr-xr-x | tools/merge.py | 13 |
1 files changed, 9 insertions, 4 deletions
diff --git a/tools/merge.py b/tools/merge.py index 974bab0..f4a007d 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -169,8 +169,8 @@ entries_to_fetch = {} for storagenode in storagenodes: print "getting new entries from", storagenode - new_entries_per_node[storagenode] = get_new_entries(storagenode) - new_entries.update(set(new_entries_per_node[storagenode])) + new_entries_per_node[storagenode] = set(get_new_entries(storagenode)) + new_entries.update(new_entries_per_node[storagenode]) entries_to_fetch[storagenode] = [] timing_point(timing, "get new entries") @@ -191,7 +191,8 @@ for hash in new_entries: added_entries = 0 for storagenode in storagenodes: - for chunk in chunks(entries_to_fetch[storagenode], 10): + print "getting", len(entries_to_fetch[storagenode]), "entries from", storagenode + for chunk in chunks(entries_to_fetch[storagenode], 100): entries = get_entries(storagenode, chunk) for hash in chunk: entry = entries[hash] @@ -231,7 +232,11 @@ for frontendnode in frontendnodes: timing_point(timing, "get curpos") print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] - sendlog(frontendnode, {"start": curpos, "hashes": entries}) + for chunk in chunks(entries, 1000): + sendlog(frontendnode, {"start": curpos, "hashes": chunk}) + curpos += len(chunk) + print curpos, + sys.stdout.flush() timing_point(timing, "sendlog") print "log sent" missingentries = get_missingentries(frontendnode) |