summaryrefslogtreecommitdiff
path: root/tools/merge_backup.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/merge_backup.py')
-rwxr-xr-x[-rw-r--r--]tools/merge_backup.py58
1 files changed, 51 insertions, 7 deletions
diff --git a/tools/merge_backup.py b/tools/merge_backup.py
index 27c71a5..48197fc 100644..100755
--- a/tools/merge_backup.py
+++ b/tools/merge_backup.py
@@ -7,12 +7,15 @@
import sys
import base64
import select
-from certtools import timing_point
+from time import sleep
+from certtools import timing_point, build_merkle_tree, write_file, \
+ create_ssl_context
from mergetools import chunks, backup_sendlog, get_logorder, \
get_verifiedsize, get_missingentriesforbackup, read_chain, \
- hexencode, setverifiedsize, sendentry_merge, verifyroot
+ hexencode, setverifiedsize, sendentry_merge, verifyroot, \
+ get_nfetched, parse_args
-def merge_backup(args, config, localconfig, sth_in):
+def merge_backup(args, config, localconfig):
paths = localconfig["paths"]
own_key = (localconfig["nodename"],
"%s/%s-private.pem" % (paths["privatekeys"],
@@ -21,12 +24,17 @@ def merge_backup(args, config, localconfig, sth_in):
mergedb = paths["mergedb"]
chainsdir = mergedb + "/chains"
logorderfile = mergedb + "/logorder"
+ currentsizefile = mergedb + "/fetched"
timing = timing_point()
- logorder = get_logorder(logorderfile)
+ nfetched = get_nfetched(currentsizefile, logorderfile)
+ logorder = get_logorder(logorderfile, nfetched)
+ tree_size = len(logorder)
timing_point(timing, "get logorder")
- (tree_size, root_hash, _) = sth_in
+ tree = build_merkle_tree(logorder)
+ root_hash = tree[-1][0]
+ timing_point(timing, "build tree")
for secondary in secondaries:
if secondary["name"] == config["primarymergenode"]:
@@ -40,6 +48,7 @@ def merge_backup(args, config, localconfig, sth_in):
timing_point(timing, "get verified size")
print >>sys.stderr, "verified size", verifiedsize
sys.stderr.flush()
+
entries = [base64.b64encode(entry) for entry in logorder[verifiedsize:]]
print >>sys.stderr, "sending log:",
sys.stderr.flush()
@@ -57,7 +66,7 @@ def merge_backup(args, config, localconfig, sth_in):
continue
break
if sendlogresult["result"] != "ok":
- print >>sys.stderr, "sendlog:", sendlogresult
+ print >>sys.stderr, "backup_sendlog:", sendlogresult
sys.exit(1)
verifiedsize += len(chunk)
print >>sys.stderr, verifiedsize,
@@ -66,11 +75,13 @@ def merge_backup(args, config, localconfig, sth_in):
timing_point(timing, "sendlog")
print >>sys.stderr, "log sent"
sys.stderr.flush()
+
missingentries = get_missingentriesforbackup(nodename, nodeaddress,
own_key, paths)
timing_point(timing, "get missing")
print >>sys.stderr, "missing entries:", len(missingentries)
sys.stderr.flush()
+
fetched_entries = 0
print >>sys.stderr, "fetching missing entries",
sys.stderr.flush()
@@ -81,7 +92,7 @@ def merge_backup(args, config, localconfig, sth_in):
read_chain(chainsdir, ehash),
ehash)
if sendentryresult["result"] != "ok":
- print >>sys.stderr, "send sth:", sendentryresult
+ print >>sys.stderr, "sendentry_merge:", sendentryresult
sys.exit(1)
fetched_entries += 1
if fetched_entries % 1000 == 0:
@@ -90,6 +101,7 @@ def merge_backup(args, config, localconfig, sth_in):
print >>sys.stderr
sys.stderr.flush()
timing_point(timing, "send missing")
+
verifyrootresult = verifyroot(nodename, nodeaddress, own_key, paths,
tree_size)
if verifyrootresult["result"] != "ok":
@@ -102,7 +114,39 @@ def merge_backup(args, config, localconfig, sth_in):
print >>sys.stderr, " expected", hexencode(root_hash)
sys.exit(1)
timing_point(timing, "verifyroot")
+
setverifiedsize(nodename, nodeaddress, own_key, paths, tree_size)
+ backuppath = mergedb + "/verified." + nodename
+ backupdata = {"tree_size": tree_size,
+ "sha256_root_hash": hexencode(root_hash)}
+ print >>sys.stderr, "DEBUG: writing to", backuppath, ":", backupdata
+ write_file(backuppath, backupdata)
+
if args.timing:
print >>sys.stderr, timing["deltatimes"]
sys.stderr.flush()
+
+def main():
+ """
+ Read logorder file up until what's indicated by fetched file and
+ build the tree.
+
+ Distribute entries to all secondaries, write tree size and tree head
+ to backup.<secondary> files as each secondary is verified to have
+ the entries.
+
+ Sleep some and start over.
+ """
+ args, config, localconfig = parse_args()
+ paths = localconfig["paths"]
+ create_ssl_context(cafile=paths["https_cacertfile"])
+
+ while True:
+ merge_backup(args, config, localconfig)
+ if args.interval is None:
+ break
+ print >>sys.stderr, "sleeping", args.interval, "seconds"
+ sleep(args.interval)
+
+if __name__ == '__main__':
+ sys.exit(main())