New entries per node should be a set. Chunk sendlog.

1 files changed, 9 insertions, 4 deletions

diff --git a/tools/merge.py b/tools/merge.py
index 974bab0..f4a007d 100755
--- a/tools/merge.py
+++ b/tools/merge.py
@@ -169,8 +169,8 @@ entries_to_fetch = {}
 
 for storagenode in storagenodes:
     print "getting new entries from", storagenode
-    new_entries_per_node[storagenode] = get_new_entries(storagenode)
-    new_entries.update(set(new_entries_per_node[storagenode]))
+    new_entries_per_node[storagenode] = set(get_new_entries(storagenode))
+    new_entries.update(new_entries_per_node[storagenode])
     entries_to_fetch[storagenode] = []
 
 timing_point(timing, "get new entries")
@@ -191,7 +191,8 @@ for hash in new_entries:
 
 added_entries = 0
 for storagenode in storagenodes:
-    for chunk in chunks(entries_to_fetch[storagenode], 10):
+    print "getting", len(entries_to_fetch[storagenode]), "entries from", storagenode
+    for chunk in chunks(entries_to_fetch[storagenode], 100):
         entries = get_entries(storagenode, chunk)
         for hash in chunk:
             entry = entries[hash]
@@ -231,7 +232,11 @@ for frontendnode in frontendnodes:
     timing_point(timing, "get curpos")
     print "current position", curpos
     entries = [base64.b64encode(entry) for entry in logorder[curpos:]]
-    sendlog(frontendnode, {"start": curpos, "hashes": entries})
+    for chunk in chunks(entries, 1000):
+        sendlog(frontendnode, {"start": curpos, "hashes": chunk})
+        curpos += len(chunk)
+        print curpos,
+        sys.stdout.flush()
     timing_point(timing, "sendlog")
     print "log sent"
     missingentries = get_missingentries(frontendnode)