From 0e58634a718acd025ac5f1ed9e385e064e530523 Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Thu, 12 Nov 2015 00:53:29 +0100
Subject: Send many entries at a time in merge_backup

---
 tools/merge_backup.py | 14 +++++++-------
 tools/mergetools.py   |  7 +++++--
 2 files changed, 12 insertions(+), 9 deletions(-)

(limited to 'tools')

diff --git a/tools/merge_backup.py b/tools/merge_backup.py
index 6834e52..123347a 100755
--- a/tools/merge_backup.py
+++ b/tools/merge_backup.py
@@ -12,7 +12,7 @@ from certtools import timing_point, build_merkle_tree, write_file, \
      create_ssl_context
 from mergetools import chunks, backup_sendlog, get_logorder, \
      get_verifiedsize, get_missingentriesforbackup, read_chain, \
-     hexencode, setverifiedsize, sendentry_merge, verifyroot, \
+     hexencode, setverifiedsize, sendentries_merge, verifyroot, \
      get_nfetched, parse_args
 
 def merge_backup(args, config, localconfig, secondaries):
@@ -84,12 +84,12 @@ def merge_backup(args, config, localconfig, secondaries):
         fetched_entries = 0
         print >>sys.stderr, "fetching missing entries",
         sys.stderr.flush()
-        for missingentry in missingentries:
-            ehash = base64.b64decode(missingentry)
-            sendentryresult = sendentry_merge(nodename, nodeaddress,
-                                              own_key, paths,
-                                              read_chain(chainsdir, ehash),
-                                              ehash)
+        for missingentry_chunk in chunks(missingentries, 100):
+            missingentry_hashes = [base64.b64decode(missingentry) for missingentry in missingentry_chunk]
+            hashes_and_entries = [(hash, read_chain(chainsdir, hash)) for hash in missingentry_hashes]
+            sendentryresult = sendentries_merge(nodename, nodeaddress,
+                                                own_key, paths,
+                                                hashes_and_entries)
             if sendentryresult["result"] != "ok":
                 print >>sys.stderr, "sendentry_merge:", sendentryresult
                 sys.exit(1)
diff --git a/tools/mergetools.py b/tools/mergetools.py
index 89ba7b2..3dbe517 100644
--- a/tools/mergetools.py
+++ b/tools/mergetools.py
@@ -306,11 +306,14 @@ def sendentry(node, baseurl, own_key, paths, entry, ehash):
         raise e
 
 def sendentry_merge(node, baseurl, own_key, paths, entry, ehash):
+    return sendentries_merge(node, baseurl, own_key, paths, [(ehash, entry)])
+
+def sendentries_merge(node, baseurl, own_key, paths, entries):
     try:
+        json_entries = [{"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)} for hash, entry in entries]
         result = http_request(
             baseurl + "plop/v1/merge/sendentry",
-            json.dumps({"entry":base64.b64encode(entry),
-                        "treeleafhash":base64.b64encode(ehash)}),
+            json.dumps(json_entries),
             key=own_key, verifynode=node, publickeydir=paths["publickeys"])
         return json.loads(result)
     except urllib2.URLError, e:
-- 
cgit v1.1