summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorMagnus Ahltorp <map@kth.se>2015-11-12 00:53:29 +0100
committerMagnus Ahltorp <map@kth.se>2015-11-12 00:53:29 +0100
commit0e58634a718acd025ac5f1ed9e385e064e530523 (patch)
treea3173e9f860e07e4ad8ab480bfbef84df4604b28 /tools
parent45b5f0500b125245dc4a10e3fcbf4c83a6bc4946 (diff)
Send many entries at a time in merge_backup
Diffstat (limited to 'tools')
-rwxr-xr-xtools/merge_backup.py14
-rw-r--r--tools/mergetools.py7
2 files changed, 12 insertions, 9 deletions
diff --git a/tools/merge_backup.py b/tools/merge_backup.py
index 6834e52..123347a 100755
--- a/tools/merge_backup.py
+++ b/tools/merge_backup.py
@@ -12,7 +12,7 @@ from certtools import timing_point, build_merkle_tree, write_file, \
create_ssl_context
from mergetools import chunks, backup_sendlog, get_logorder, \
get_verifiedsize, get_missingentriesforbackup, read_chain, \
- hexencode, setverifiedsize, sendentry_merge, verifyroot, \
+ hexencode, setverifiedsize, sendentries_merge, verifyroot, \
get_nfetched, parse_args
def merge_backup(args, config, localconfig, secondaries):
@@ -84,12 +84,12 @@ def merge_backup(args, config, localconfig, secondaries):
fetched_entries = 0
print >>sys.stderr, "fetching missing entries",
sys.stderr.flush()
- for missingentry in missingentries:
- ehash = base64.b64decode(missingentry)
- sendentryresult = sendentry_merge(nodename, nodeaddress,
- own_key, paths,
- read_chain(chainsdir, ehash),
- ehash)
+ for missingentry_chunk in chunks(missingentries, 100):
+ missingentry_hashes = [base64.b64decode(missingentry) for missingentry in missingentry_chunk]
+ hashes_and_entries = [(hash, read_chain(chainsdir, hash)) for hash in missingentry_hashes]
+ sendentryresult = sendentries_merge(nodename, nodeaddress,
+ own_key, paths,
+ hashes_and_entries)
if sendentryresult["result"] != "ok":
print >>sys.stderr, "sendentry_merge:", sendentryresult
sys.exit(1)
diff --git a/tools/mergetools.py b/tools/mergetools.py
index 89ba7b2..3dbe517 100644
--- a/tools/mergetools.py
+++ b/tools/mergetools.py
@@ -306,11 +306,14 @@ def sendentry(node, baseurl, own_key, paths, entry, ehash):
raise e
def sendentry_merge(node, baseurl, own_key, paths, entry, ehash):
+ return sendentries_merge(node, baseurl, own_key, paths, [(ehash, entry)])
+
+def sendentries_merge(node, baseurl, own_key, paths, entries):
try:
+ json_entries = [{"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)} for hash, entry in entries]
result = http_request(
baseurl + "plop/v1/merge/sendentry",
- json.dumps({"entry":base64.b64encode(entry),
- "treeleafhash":base64.b64encode(ehash)}),
+ json.dumps(json_entries),
key=own_key, verifynode=node, publickeydir=paths["publickeys"])
return json.loads(result)
except urllib2.URLError, e: