summaryrefslogtreecommitdiff
path: root/tools/merge_fetch.py
diff options
context:
space:
mode:
authorLinus Nordberg <linus@nordu.net>2015-09-24 16:47:32 +0200
committerLinus Nordberg <linus@nordu.net>2015-11-10 12:48:46 +0100
commitaaed8aff7c425c8ac2a15e584e24317da327f5e4 (patch)
tree229a7e5718b0d4c9750918d654484dba354f5194 /tools/merge_fetch.py
parentab327d88f7a8f458b6150efd6b21b5615210e571 (diff)
Merge is now run by shell script tools/merge.
tools/merge run merge_fetch.py, merge_backup.py, merge_sth.py and merge_dist.py sequentially. TODO: test backupquorum != 0
Diffstat (limited to 'tools/merge_fetch.py')
-rwxr-xr-x[-rw-r--r--]tools/merge_fetch.py48
1 files changed, 36 insertions, 12 deletions
diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py
index a0a0396..1540c34 100644..100755
--- a/tools/merge_fetch.py
+++ b/tools/merge_fetch.py
@@ -6,13 +6,14 @@
import sys
import struct
-import time
import subprocess
+from time import sleep
from mergetools import get_logorder, verify_entry, get_new_entries, \
- chunks, fsync_logorder, get_entries, write_chain, add_to_logorder
-from certtools import timing_point, build_merkle_tree
+ chunks, fsync_logorder, get_entries, write_chain, add_to_logorder, \
+ hexencode, parse_args
+from certtools import timing_point, write_file, create_ssl_context
-def merge_fetch(args, config, localconfig):
+def merge_fetch(_args, config, localconfig):
paths = localconfig["paths"]
storagenodes = config["storagenodes"]
mergedb = paths["mergedb"]
@@ -21,7 +22,6 @@ def merge_fetch(args, config, localconfig):
own_key = (localconfig["nodename"],
"%s/%s-private.pem" % (paths["privatekeys"],
localconfig["nodename"]))
-
timing = timing_point()
logorder = get_logorder(logorderfile)
@@ -48,9 +48,6 @@ def merge_fetch(args, config, localconfig):
print >>sys.stderr, "adding", len(new_entries), "entries"
sys.stderr.flush()
- if args.nomerge:
- sys.exit(0)
-
for ehash in new_entries:
for storagenode in storagenodes:
if ehash in new_entries_per_node[storagenode["name"]]:
@@ -89,9 +86,36 @@ def merge_fetch(args, config, localconfig):
verifycert.communicate(struct.pack("I", 0))
- tree = build_merkle_tree(logorder)
tree_size = len(logorder)
- root_hash = tree[-1][0]
- timestamp = int(time.time() * 1000)
+ if tree_size == 0:
+ return (0, '')
+ else:
+ return (tree_size, logorder[tree_size-1])
+
+def main():
+ """
+ Fetch new entries from all storage nodes.
+
+ Indicate current position by writing the index in the logorder file
+ (0-based) to the 'fetched' file.
+
+ Sleep some and start over.
+ """
+ args, config, localconfig = parse_args()
+ paths = localconfig["paths"]
+ mergedb = paths["mergedb"]
+ currentsizefile = mergedb + "/fetched"
+ create_ssl_context(cafile=paths["https_cacertfile"])
+
+ while True:
+ logsize, last_hash = merge_fetch(args, config, localconfig)
+ currentsize = {"index": logsize - 1, "hash": hexencode(last_hash)}
+ print >>sys.stderr, "DEBUG: writing to", currentsizefile, ":", currentsize
+ write_file(currentsizefile, currentsize)
+ if args.interval is None:
+ break
+ print >>sys.stderr, "sleeping", args.interval, "seconds"
+ sleep(args.interval)
- return (tree_size, root_hash, timestamp)
+if __name__ == '__main__':
+ sys.exit(main())