Manual rebase onto master with map/robust-distribution.

1 files changed, 54 insertions, 27 deletions

diff --git a/tools/merge_sth.py b/tools/merge_sth.py
index f4aec53..97f6e24 100755
--- a/tools/merge_sth.py
+++ b/tools/merge_sth.py
@@ -9,12 +9,13 @@
 #
 import sys
 import json
-import urllib2
 import time
 import requests
+import logging
 from base64 import b64encode
+from datetime import datetime, timedelta
 from mergetools import parse_args, get_nfetched, hexencode, hexdecode, \
-     get_logorder, get_sth
+     get_logorder, get_sth, flock_ex_or_fail
 from certtools import create_ssl_context, get_public_key_from_file, \
      timing_point, create_sth_signature, write_file, check_sth_signature, \
      build_merkle_tree
@@ -39,6 +40,7 @@ def merge_sth(args, config, localconfig):
 
     trees = [{'tree_size': get_nfetched(currentsizefile, logorderfile),
               'sha256_root_hash': ''}]
+    logging.debug("starting point, trees: %s", trees)
     for mergenode in mergenodes:
         if mergenode["name"] == config["primarymergenode"]:
             continue
@@ -49,28 +51,29 @@ def merge_sth(args, config, localconfig):
             tree = {'tree_size': 0, "sha256_root_hash": ''}
         trees.append(tree)
     trees.sort(key=lambda e: e['tree_size'], reverse=True)
-    #print >>sys.stderr, "DEBUG: trees:", trees
+    logging.debug("trees: %s", trees)
 
     if backupquorum > len(trees) - 1:
-        print >>sys.stderr, "backup quorum > number of secondaries:", \
-          backupquorum, ">", len(trees) - 1
-        return
+        logging.error("backup quorum > number of secondaries: %d > %d",
+                      backupquorum, len(trees) - 1)
+        return -1
     tree_size = trees[backupquorum]['tree_size']
     root_hash = hexdecode(trees[backupquorum]['sha256_root_hash'])
-    #print >>sys.stderr, "DEBUG: tree size candidate at backupquorum", backupquorum, ":", tree_size
+    logging.debug("tree size candidate at backupquorum %d: %d", backupquorum,
+                  tree_size)
 
     cur_sth = get_sth(sthfile)
     if tree_size < cur_sth['tree_size']:
-        print >>sys.stderr, "candidate tree < current tree:", \
-          tree_size, "<", cur_sth['tree_size']
-        return
+        logging.info("candidate tree < current tree: %d < %d",
+                     tree_size, cur_sth['tree_size'])
+        return 0
 
     assert tree_size >= 0         # Don't read logorder without limit.
     logorder = get_logorder(logorderfile, tree_size)
     timing_point(timing, "get logorder")
     if tree_size == -1:
         tree_size = len(logorder)
-    print >>sys.stderr, "new tree size will be", tree_size
+    logging.info("new tree size will be %d", tree_size)
 
     root_hash_calc = build_merkle_tree(logorder)[-1][0]
     assert root_hash == '' or root_hash == root_hash_calc
@@ -87,11 +90,10 @@ def merge_sth(args, config, localconfig):
                                    key=own_key)
             break
         except requests.exceptions.HTTPError, e:
-            print >>sys.stderr, e.response
-            sys.stderr.flush()
+            logging.warning("create_sth_signature error: %s", e.response)
     if tree_head_signature == None:
-        print >>sys.stderr, "Could not contact any signing nodes"
-        sys.exit(1)
+        logging.error("Could not contact any signing nodes")
+        return 0
 
     sth = {"tree_size": tree_size, "timestamp": timestamp,
            "sha256_root_hash": b64encode(root_hash),
@@ -100,34 +102,59 @@ def merge_sth(args, config, localconfig):
     check_sth_signature(ctbaseurl, sth, publickey=logpublickey)
     timing_point(timing, "build sth")
 
-    print hexencode(root_hash), timestamp, tree_size
-    sys.stdout.flush()
+    logging.info("new root: %s %d %d", hexencode(root_hash), timestamp, tree_size)
 
     write_file(sthfile, sth)
 
     if args.timing:
-        print >>sys.stderr, "timing: merge_sth:", timing["deltatimes"]
-        sys.stderr.flush()
+        logging.debug("timing: merge_sth: %s", timing["deltatimes"])
+
+    return 0
 
 def main():
     """
-    Read file 'sth' to get current tree size, assuming zero if file not
+    Read 'sth' to get the current tree size, assuming zero if file not
     found.
 
     Read tree sizes from the backup.<secondary> files, put them in a
-    list and sort it. Let new tree size equal list[backup-quorum]. Barf
-    on a new tree size smaller than the currently published tree size.
+    list and sort the list. Let new tree size be list[backup-quorum]. If
+    the new tree size is smaller than the currently published tree size,
+    stop here.
+
+    Decide on a timestamp, build an STH and write it to 'sth'.
 
-    Decide on a timestamp, build an STH and write it to file 'sth'.
+    Sleep some and start over, or exit if there's no `--mergeinterval'.
     """
     args, config, localconfig = parse_args()
+    paths = localconfig["paths"]
+    mergedb = paths["mergedb"]
+    lockfile = mergedb + "/.merge_sth.lock"
+
+    loglevel = getattr(logging, args.loglevel.upper())
+    if args.mergeinterval is None:
+        logging.basicConfig(level=loglevel)
+    else:
+        logging.basicConfig(filename=args.logdir + "/merge_sth.log",
+                            level=loglevel)
+
+    if not flock_ex_or_fail(lockfile):
+        logging.critical("unable to take lock %s", lockfile)
+        return 1
 
     while True:
-        merge_sth(args, config, localconfig)
-        if args.interval is None:
+        merge_start_time = datetime.now()
+        ret = merge_sth(args, config, localconfig)
+        if ret < 0:
+            return 1
+        if args.mergeinterval is None:
             break
-        print >>sys.stderr, "sleeping", args.interval, "seconds"
-        time.sleep(args.interval)
+        sleep = (merge_start_time + timedelta(seconds=args.mergeinterval) -
+                 datetime.now()).seconds
+        if sleep > 0:
+            logging.debug("sleeping %d seconds", sleep)
+            time.sleep(sleep)
+
+    return 0
 
 if __name__ == '__main__':
     sys.exit(main())