#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright (c) 2014-2015, NORDUnet A/S. # See LICENSE for licensing information. import sys import urllib2 import base64 import select from certtools import timing_point, check_sth_signature, \ create_sth_signature, get_public_key_from_file from mergetools import get_curpos, get_logorder, chunks, get_missingentries, \ sendsth, hexencode, sendlog, sendentry, read_chain def merge_dist(args, config, localconfig, sth_in): paths = localconfig["paths"] own_key = (localconfig["nodename"], "%s/%s-private.pem" % (paths["privatekeys"], localconfig["nodename"])) frontendnodes = config["frontendnodes"] signingnodes = config["signingnodes"] ctbaseurl = config["baseurl"] logpublickey = get_public_key_from_file(paths["logpublickey"]) mergedb = paths["mergedb"] chainsdir = mergedb + "/chains" logorderfile = mergedb + "/logorder" timing = timing_point() logorder = get_logorder(logorderfile) timing_point(timing, "get logorder") (tree_size, root_hash, timestamp) = sth_in tree_head_signature = None for signingnode in signingnodes: try: tree_head_signature = \ create_sth_signature(tree_size, timestamp, root_hash, "https://%s/" % signingnode["address"], key=own_key) break except urllib2.URLError, err: print >>sys.stderr, err sys.stderr.flush() if tree_head_signature == None: print >>sys.stderr, "Could not contact any signing nodes" sys.exit(1) sth = {"tree_size": tree_size, "timestamp": timestamp, "sha256_root_hash": base64.b64encode(root_hash), "tree_head_signature": base64.b64encode(tree_head_signature)} check_sth_signature(ctbaseurl, sth, publickey=logpublickey) timing_point(timing, "build sth") if args.timing: print >>sys.stderr, timing["deltatimes"] sys.stderr.flush() print hexencode(root_hash) sys.stdout.flush() for frontendnode in frontendnodes: nodeaddress = "https://%s/" % frontendnode["address"] nodename = frontendnode["name"] timing = timing_point() print >>sys.stderr, "distributing for node", nodename sys.stderr.flush() curpos = get_curpos(nodename, nodeaddress, own_key, paths) timing_point(timing, "get curpos") print >>sys.stderr, "current position", curpos sys.stderr.flush() entries = [base64.b64encode(entry) for entry in logorder[curpos:]] print >>sys.stderr, "sending log:", sys.stderr.flush() for chunk in chunks(entries, 1000): for trynumber in range(5, 0, -1): sendlogresult = sendlog(nodename, nodeaddress, own_key, paths, {"start": curpos, "hashes": chunk}) if sendlogresult == None: if trynumber == 1: sys.exit(1) select.select([], [], [], 10.0) print >>sys.stderr, "tries left:", trynumber sys.stderr.flush() continue break if sendlogresult["result"] != "ok": print >>sys.stderr, "sendlog:", sendlogresult sys.exit(1) curpos += len(chunk) print >>sys.stderr, curpos, sys.stderr.flush() print >>sys.stderr timing_point(timing, "sendlog") print >>sys.stderr, "log sent" sys.stderr.flush() missingentries = get_missingentries(nodename, nodeaddress, own_key, paths) timing_point(timing, "get missing") print >>sys.stderr, "missing entries:", len(missingentries) sys.stderr.flush() fetched_entries = 0 print >>sys.stderr, "fetching missing entries", sys.stderr.flush() for missingentry in missingentries: ehash = base64.b64decode(missingentry) sendentryresult = sendentry(nodename, nodeaddress, own_key, paths, read_chain(chainsdir, ehash), ehash) if sendentryresult["result"] != "ok": print >>sys.stderr, "send sth:", sendentryresult sys.exit(1) fetched_entries += 1 if fetched_entries % 1000 == 0: print >>sys.stderr, fetched_entries, sys.stderr.flush() print >>sys.stderr sys.stderr.flush() timing_point(timing, "send missing") sendsthresult = sendsth(nodename, nodeaddress, own_key, paths, sth) if sendsthresult["result"] != "ok": print >>sys.stderr, "send sth:", sendsthresult sys.exit(1) timing_point(timing, "send sth") if args.timing: print >>sys.stderr, timing["deltatimes"] sys.stderr.flush()