diff options
| -rwxr-xr-x | test/scripts/light-system-test-prepare-merge-takeover.sh | 4 | ||||
| -rwxr-xr-x | test/scripts/light-system-test-start.sh | 11 | ||||
| -rwxr-xr-x | test/scripts/light-system-test-stop.sh | 7 | ||||
| -rwxr-xr-x | test/scripts/light-system-test.sh | 7 | ||||
| -rw-r--r-- | test/scripts/testutils.sh | 4 | ||||
| -rw-r--r-- | tools/certtools.py | 18 | ||||
| -rwxr-xr-x | tools/compileconfig.py | 25 | ||||
| -rwxr-xr-x | tools/initlog.py | 30 | ||||
| -rwxr-xr-x | tools/merge | 6 |
9 files changed, 86 insertions, 26 deletions
diff --git a/test/scripts/light-system-test-prepare-merge-takeover.sh b/test/scripts/light-system-test-prepare-merge-takeover.sh index 67e5ac7..4b50ad7 100755 --- a/test/scripts/light-system-test-prepare-merge-takeover.sh +++ b/test/scripts/light-system-test-prepare-merge-takeover.sh @@ -9,8 +9,8 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd) mv mergedb mergedb-down mv mergedb-secondary mergedb -init_log -cp mergedb-down/sth mergedb/ # After init_log. +mv mergedb/verifiedsize mergedb/verifiedsize.OFF +cp mergedb-down/logorder mergedb/ mkdir mergedb-secondary touch mergedb-secondary/logorder printf 0 > mergedb-secondary/verifiedsize diff --git a/test/scripts/light-system-test-start.sh b/test/scripts/light-system-test-start.sh index 4520945..985a23c 100755 --- a/test/scripts/light-system-test-start.sh +++ b/test/scripts/light-system-test-start.sh @@ -14,8 +14,11 @@ start_node() { "exec ../bin/erl -boot ${node_app} -config ${node}" } -for node in ${ERLANGNODES}; do - start_node $(echo $node | tr ':' ' ') +for nodegroup in $SIGNINGNODES $MERGESECONDARYNODES $STORAGENODES $FRONTENDNODES; do + for node in $nodegroup; do + echo "starting $node" + start_node $(echo $node | tr ':' ' ') + done done for i in 1 2 3 4 5 6 7 8 9 10; do echo "waiting for system to start" @@ -37,3 +40,7 @@ for i in 1 2 3 4 5 6 7 8 9 10; do echo Not started: ${notstarted} fi done + +[ $# -gt 0 ] && [ "$1" = "initlog" ] && $top_srcdir/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg + +start_node $(echo ${MERGEPRIMARYNODE} | tr ':' ' ') diff --git a/test/scripts/light-system-test-stop.sh b/test/scripts/light-system-test-stop.sh index 0bf4374..2265a34 100755 --- a/test/scripts/light-system-test-stop.sh +++ b/test/scripts/light-system-test-stop.sh @@ -12,6 +12,9 @@ stop_node() { ${top_srcdir}/tools/to_catlfish.py to_erl nodes/${node}/ "init:stop()" } -for node in ${ERLANGNODES}; do - stop_node $(echo $node | tr ':' ' ') +for nodegroup in $FRONTENDNODES $MERGEPRIMARYNODE $STORAGENODES $MERGESECONDARYNODES $SIGNINGNODES; do + for node in $nodegroup; do + echo "starting $node" + stop_node $(echo $node | tr ':' ' ') + done done diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh index 1f761e8..d2d242c 100755 --- a/test/scripts/light-system-test.sh +++ b/test/scripts/light-system-test.sh @@ -10,7 +10,7 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd) SCRIPTS=${top_srcdir}/test/scripts tests_start() { - ${SCRIPTS}/light-system-test-start.sh + ${SCRIPTS}/light-system-test-start.sh $@ } tests_stop() { @@ -18,8 +18,7 @@ tests_stop() { } ${SCRIPTS}/light-system-test-prepare.sh -init_log -tests_start +tests_start initlog ${SCRIPTS}/light-system-test-run-1.sh || (echo "Tests failed"; sleep 5; tests_stop; false) sleep 5 tests_stop @@ -50,7 +49,7 @@ echo echo echo ${SCRIPTS}/light-system-test-prepare-merge-takeover.sh -tests_start +tests_start initlog ${SCRIPTS}/light-system-test-run-3.sh || (echo "Tests failed"; sleep 5; tests_stop; false) sleep 5 tests_stop diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh index 657607b..88e9a54 100644 --- a/test/scripts/testutils.sh +++ b/test/scripts/testutils.sh @@ -47,7 +47,3 @@ do_merge() { sleep 1 done } - -init_log() { - ${top_srcdir}/tools/initlog.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Log init failed" -} diff --git a/tools/certtools.py b/tools/certtools.py index 9febc69..82339aa 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -761,3 +761,21 @@ def write_file(fn, jsondata): tempname = fn + ".new" open(tempname, 'w').write(json.dumps(jsondata)) mv_file(tempname, fn) + +def generate_tree_head_signature(signingnodes, own_key, + tree_size, timestamp, root_hash): + tree_head_signature = None + for signingnode in signingnodes: + try: + tree_head_signature = \ + create_sth_signature(tree_size, timestamp, + root_hash, + "https://%s/" % signingnode["address"], + key=own_key) + break + except requests.exceptions.HTTPError, e: + logging.warning("create_sth_signature error: %s", e.response) + if tree_head_signature == None: + logging.error("Could not contact any signing nodes") + + return tree_head_signature diff --git a/tools/compileconfig.py b/tools/compileconfig.py index abe59a3..a5f013a 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -186,7 +186,7 @@ def allowed_servers_frontend(signingnodenames, storagenodenames): ("/plop/v1/signing/sct", signingnodenames), ] -def allowed_servers_primarymerge(frontendnodenames, secondarymergenames): +def allowed_servers_primarymerge(frontendnodenames, secondarymergenames, signingnodenames): return [ ("/plop/v1/frontend/verify-entries", frontendnodenames), ("/plop/v1/frontend/sendlog", frontendnodenames), @@ -197,6 +197,7 @@ def allowed_servers_primarymerge(frontendnodenames, secondarymergenames): ("/plop/v1/merge/setverifiedsize", secondarymergenames), ("/plop/v1/merge/sendlog", secondarymergenames), ("/plop/v1/merge/sendentry", secondarymergenames), + ("/plop/v1/signing/sth", signingnodenames), ] def parse_ratelimit_expression(expression): @@ -327,7 +328,7 @@ def gen_config(nodename, config, localconfig): (Symbol("entry_root_path"), paths["mergedb"] + "/chains"), ] - signingnodes = config["signingnodes"] + signingnodenames = [node["name"] for node in config["signingnodes"]] signingnodeaddresses = ["https://%s/plop/v1/signing/" % node["address"] for node in config["signingnodes"]] mergenodenames = [node["name"] for node in config["mergenodes"]] primarymergenodename = config["primarymergenode"] @@ -346,7 +347,7 @@ def gen_config(nodename, config, localconfig): services.add(Symbol("ht")) allowed_clients += allowed_clients_frontend(mergenodenames, primarymergenodename) allowed_clients += allowed_clients_public() - allowed_servers += allowed_servers_frontend([node["name"] for node in signingnodes], storagenodenames) + allowed_servers += allowed_servers_frontend(signingnodenames, storagenodenames) if "storagenodes" in nodetype: allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames) if "signingnodes" in nodetype: @@ -373,7 +374,9 @@ def gen_config(nodename, config, localconfig): plopconfig.append((Symbol("sth_path"), paths["mergedb"] + "/sth")) plopconfig.append((Symbol("fetched_path"), paths["mergedb"] + "/fetched")) plopconfig.append((Symbol("verified_path"), paths["mergedb"] + "/verified")) - allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames) + reloadableplopconfig.append((Symbol("signing_nodes"), signingnodeaddresses)) + reloadableplopconfig.append((Symbol("backup_quorum"), config["backup-quorum-size"])) + allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames, signingnodenames) else: allowed_clients += allowed_clients_mergesecondary(primarymergenodename) @@ -429,19 +432,25 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False): storagenodenames = set([node["name"] for node in config["storagenodes"]]) signingnodenames = set([node["name"] for node in config["signingnodes"]]) mergenodenames = set([node["name"] for node in config["mergenodes"]]) - erlangnodenames_and_apps = ['%s:%s' % (nn, 'catlfish' if nn != config["primarymergenode"] else "merge") for nn in frontendnodenames | storagenodenames | signingnodenames | mergenodenames] + mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]) frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]] storagenodeaddresses = [node["address"] for node in config["storagenodes"]] signingnodeaddresses = [node["address"] for node in config["signingnodes"]] - mergenodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]] + mergesecondarynodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]] delimiter = '"' if shellvars else '' print >>configfile, "NODES=" + delimiter + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + delimiter - print >>configfile, "ERLANGNODES=" + delimiter + " ".join(erlangnodenames_and_apps) + delimiter + + print >>configfile, "FRONTENDNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in frontendnodenames]) + delimiter + print >>configfile, "STORAGENODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in storagenodenames if name not in frontendnodenames]) + delimiter + print >>configfile, "SIGNINGNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in signingnodenames]) + delimiter + print >>configfile, "MERGESECONDARYNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in mergesecondarynodenames]) + delimiter + print >>configfile, "MERGEPRIMARYNODE=" '%s:%s' % (config["primarymergenode"], 'merge') + print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter - print >>configfile, "TESTURLS=" + delimiter + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergenodeaddresses) + delimiter + print >>configfile, "TESTURLS=" + delimiter + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergesecondarynodeaddresses) + delimiter print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter configfile.close() diff --git a/tools/initlog.py b/tools/initlog.py index 11ebc2e..3ef5720 100755 --- a/tools/initlog.py +++ b/tools/initlog.py @@ -14,7 +14,7 @@ import yaml import errno from time import time from base64 import b64encode -from certtools import build_merkle_tree, write_file +from certtools import build_merkle_tree, write_file, generate_tree_head_signature from mergetools import get_sth, perm, get_logorder def parse_args(): @@ -33,6 +33,9 @@ def parse_args(): def main(): """ Initialise a log by creating + - sth file + - must not exist before + - consulting 'logorder' if it exists - perm database if it doesn't exist """ args, config, localconfig = parse_args() @@ -65,6 +68,31 @@ def main(): # Create a chains database. chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains") + # Create sth file. + tree_size = 0 + root_hash = build_merkle_tree('')[-1][0] + try: + logorder = get_logorder(logorderfile) + tree_size = len(logorder) + root_hash = build_merkle_tree(logorder[:tree_size])[-1][0] + except IOError, e: + if e.errno == errno.ENOENT: + pass + timestamp = int(time() * 1000) + tree_head_signature = \ + generate_tree_head_signature(signingnodes, own_key, + tree_size, timestamp, root_hash) + if tree_head_signature == None: + return 1 + + sth = {"tree_size": tree_size, + "timestamp": timestamp, + "sha256_root_hash": b64encode(root_hash), + "tree_head_signature": b64encode(tree_head_signature)} + print "Creating sth file with tree size", tree_size, \ + "and timestamp", timestamp + write_file(sthfile, sth) + return 0 if __name__ == '__main__': diff --git a/tools/merge b/tools/merge index d0198ec..19e22f8 100755 --- a/tools/merge +++ b/tools/merge @@ -5,6 +5,6 @@ set -o errexit BINDIR=$(dirname $0) $BINDIR/merge_fetch.py "$@" -#$BINDIR/merge_backup.py "$@" -$BINDIR/merge_sth.py "$@" -#$BINDIR/merge_dist.py "$@" +# $BINDIR/merge_backup.py "$@" +# $BINDIR/merge_sth.py "$@" +# $BINDIR/merge_dist.py "$@" |