summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xtest/scripts/light-system-test-prepare-merge-takeover.sh4
-rwxr-xr-xtest/scripts/light-system-test-start.sh11
-rwxr-xr-xtest/scripts/light-system-test-stop.sh7
-rwxr-xr-xtest/scripts/light-system-test.sh7
-rw-r--r--test/scripts/testutils.sh4
-rw-r--r--tools/certtools.py18
-rwxr-xr-xtools/compileconfig.py25
-rwxr-xr-xtools/initlog.py30
-rwxr-xr-xtools/merge6
9 files changed, 86 insertions, 26 deletions
diff --git a/test/scripts/light-system-test-prepare-merge-takeover.sh b/test/scripts/light-system-test-prepare-merge-takeover.sh
index 67e5ac7..4b50ad7 100755
--- a/test/scripts/light-system-test-prepare-merge-takeover.sh
+++ b/test/scripts/light-system-test-prepare-merge-takeover.sh
@@ -9,8 +9,8 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
mv mergedb mergedb-down
mv mergedb-secondary mergedb
-init_log
-cp mergedb-down/sth mergedb/ # After init_log.
+mv mergedb/verifiedsize mergedb/verifiedsize.OFF
+cp mergedb-down/logorder mergedb/
mkdir mergedb-secondary
touch mergedb-secondary/logorder
printf 0 > mergedb-secondary/verifiedsize
diff --git a/test/scripts/light-system-test-start.sh b/test/scripts/light-system-test-start.sh
index 4520945..985a23c 100755
--- a/test/scripts/light-system-test-start.sh
+++ b/test/scripts/light-system-test-start.sh
@@ -14,8 +14,11 @@ start_node() {
"exec ../bin/erl -boot ${node_app} -config ${node}"
}
-for node in ${ERLANGNODES}; do
- start_node $(echo $node | tr ':' ' ')
+for nodegroup in $SIGNINGNODES $MERGESECONDARYNODES $STORAGENODES $FRONTENDNODES; do
+ for node in $nodegroup; do
+ echo "starting $node"
+ start_node $(echo $node | tr ':' ' ')
+ done
done
for i in 1 2 3 4 5 6 7 8 9 10; do
echo "waiting for system to start"
@@ -37,3 +40,7 @@ for i in 1 2 3 4 5 6 7 8 9 10; do
echo Not started: ${notstarted}
fi
done
+
+[ $# -gt 0 ] && [ "$1" = "initlog" ] && $top_srcdir/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+
+start_node $(echo ${MERGEPRIMARYNODE} | tr ':' ' ')
diff --git a/test/scripts/light-system-test-stop.sh b/test/scripts/light-system-test-stop.sh
index 0bf4374..2265a34 100755
--- a/test/scripts/light-system-test-stop.sh
+++ b/test/scripts/light-system-test-stop.sh
@@ -12,6 +12,9 @@ stop_node() {
${top_srcdir}/tools/to_catlfish.py to_erl nodes/${node}/ "init:stop()"
}
-for node in ${ERLANGNODES}; do
- stop_node $(echo $node | tr ':' ' ')
+for nodegroup in $FRONTENDNODES $MERGEPRIMARYNODE $STORAGENODES $MERGESECONDARYNODES $SIGNINGNODES; do
+ for node in $nodegroup; do
+ echo "starting $node"
+ stop_node $(echo $node | tr ':' ' ')
+ done
done
diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh
index 1f761e8..d2d242c 100755
--- a/test/scripts/light-system-test.sh
+++ b/test/scripts/light-system-test.sh
@@ -10,7 +10,7 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
SCRIPTS=${top_srcdir}/test/scripts
tests_start() {
- ${SCRIPTS}/light-system-test-start.sh
+ ${SCRIPTS}/light-system-test-start.sh $@
}
tests_stop() {
@@ -18,8 +18,7 @@ tests_stop() {
}
${SCRIPTS}/light-system-test-prepare.sh
-init_log
-tests_start
+tests_start initlog
${SCRIPTS}/light-system-test-run-1.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
sleep 5
tests_stop
@@ -50,7 +49,7 @@ echo
echo
echo
${SCRIPTS}/light-system-test-prepare-merge-takeover.sh
-tests_start
+tests_start initlog
${SCRIPTS}/light-system-test-run-3.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
sleep 5
tests_stop
diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh
index 657607b..88e9a54 100644
--- a/test/scripts/testutils.sh
+++ b/test/scripts/testutils.sh
@@ -47,7 +47,3 @@ do_merge() {
sleep 1
done
}
-
-init_log() {
- ${top_srcdir}/tools/initlog.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Log init failed"
-}
diff --git a/tools/certtools.py b/tools/certtools.py
index 9febc69..82339aa 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -761,3 +761,21 @@ def write_file(fn, jsondata):
tempname = fn + ".new"
open(tempname, 'w').write(json.dumps(jsondata))
mv_file(tempname, fn)
+
+def generate_tree_head_signature(signingnodes, own_key,
+ tree_size, timestamp, root_hash):
+ tree_head_signature = None
+ for signingnode in signingnodes:
+ try:
+ tree_head_signature = \
+ create_sth_signature(tree_size, timestamp,
+ root_hash,
+ "https://%s/" % signingnode["address"],
+ key=own_key)
+ break
+ except requests.exceptions.HTTPError, e:
+ logging.warning("create_sth_signature error: %s", e.response)
+ if tree_head_signature == None:
+ logging.error("Could not contact any signing nodes")
+
+ return tree_head_signature
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index abe59a3..a5f013a 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -186,7 +186,7 @@ def allowed_servers_frontend(signingnodenames, storagenodenames):
("/plop/v1/signing/sct", signingnodenames),
]
-def allowed_servers_primarymerge(frontendnodenames, secondarymergenames):
+def allowed_servers_primarymerge(frontendnodenames, secondarymergenames, signingnodenames):
return [
("/plop/v1/frontend/verify-entries", frontendnodenames),
("/plop/v1/frontend/sendlog", frontendnodenames),
@@ -197,6 +197,7 @@ def allowed_servers_primarymerge(frontendnodenames, secondarymergenames):
("/plop/v1/merge/setverifiedsize", secondarymergenames),
("/plop/v1/merge/sendlog", secondarymergenames),
("/plop/v1/merge/sendentry", secondarymergenames),
+ ("/plop/v1/signing/sth", signingnodenames),
]
def parse_ratelimit_expression(expression):
@@ -327,7 +328,7 @@ def gen_config(nodename, config, localconfig):
(Symbol("entry_root_path"), paths["mergedb"] + "/chains"),
]
- signingnodes = config["signingnodes"]
+ signingnodenames = [node["name"] for node in config["signingnodes"]]
signingnodeaddresses = ["https://%s/plop/v1/signing/" % node["address"] for node in config["signingnodes"]]
mergenodenames = [node["name"] for node in config["mergenodes"]]
primarymergenodename = config["primarymergenode"]
@@ -346,7 +347,7 @@ def gen_config(nodename, config, localconfig):
services.add(Symbol("ht"))
allowed_clients += allowed_clients_frontend(mergenodenames, primarymergenodename)
allowed_clients += allowed_clients_public()
- allowed_servers += allowed_servers_frontend([node["name"] for node in signingnodes], storagenodenames)
+ allowed_servers += allowed_servers_frontend(signingnodenames, storagenodenames)
if "storagenodes" in nodetype:
allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames)
if "signingnodes" in nodetype:
@@ -373,7 +374,9 @@ def gen_config(nodename, config, localconfig):
plopconfig.append((Symbol("sth_path"), paths["mergedb"] + "/sth"))
plopconfig.append((Symbol("fetched_path"), paths["mergedb"] + "/fetched"))
plopconfig.append((Symbol("verified_path"), paths["mergedb"] + "/verified"))
- allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames)
+ reloadableplopconfig.append((Symbol("signing_nodes"), signingnodeaddresses))
+ reloadableplopconfig.append((Symbol("backup_quorum"), config["backup-quorum-size"]))
+ allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames, signingnodenames)
else:
allowed_clients += allowed_clients_mergesecondary(primarymergenodename)
@@ -429,19 +432,25 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False):
storagenodenames = set([node["name"] for node in config["storagenodes"]])
signingnodenames = set([node["name"] for node in config["signingnodes"]])
mergenodenames = set([node["name"] for node in config["mergenodes"]])
- erlangnodenames_and_apps = ['%s:%s' % (nn, 'catlfish' if nn != config["primarymergenode"] else "merge") for nn in frontendnodenames | storagenodenames | signingnodenames | mergenodenames]
+ mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]])
frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]]
storagenodeaddresses = [node["address"] for node in config["storagenodes"]]
signingnodeaddresses = [node["address"] for node in config["signingnodes"]]
- mergenodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]
+ mergesecondarynodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]
delimiter = '"' if shellvars else ''
print >>configfile, "NODES=" + delimiter + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + delimiter
- print >>configfile, "ERLANGNODES=" + delimiter + " ".join(erlangnodenames_and_apps) + delimiter
+
+ print >>configfile, "FRONTENDNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in frontendnodenames]) + delimiter
+ print >>configfile, "STORAGENODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in storagenodenames if name not in frontendnodenames]) + delimiter
+ print >>configfile, "SIGNINGNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in signingnodenames]) + delimiter
+ print >>configfile, "MERGESECONDARYNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in mergesecondarynodenames]) + delimiter
+ print >>configfile, "MERGEPRIMARYNODE=" '%s:%s' % (config["primarymergenode"], 'merge')
+
print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter
- print >>configfile, "TESTURLS=" + delimiter + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergenodeaddresses) + delimiter
+ print >>configfile, "TESTURLS=" + delimiter + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergesecondarynodeaddresses) + delimiter
print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter
configfile.close()
diff --git a/tools/initlog.py b/tools/initlog.py
index 11ebc2e..3ef5720 100755
--- a/tools/initlog.py
+++ b/tools/initlog.py
@@ -14,7 +14,7 @@ import yaml
import errno
from time import time
from base64 import b64encode
-from certtools import build_merkle_tree, write_file
+from certtools import build_merkle_tree, write_file, generate_tree_head_signature
from mergetools import get_sth, perm, get_logorder
def parse_args():
@@ -33,6 +33,9 @@ def parse_args():
def main():
"""
Initialise a log by creating
+ - sth file
+ - must not exist before
+ - consulting 'logorder' if it exists
- perm database if it doesn't exist
"""
args, config, localconfig = parse_args()
@@ -65,6 +68,31 @@ def main():
# Create a chains database.
chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains")
+ # Create sth file.
+ tree_size = 0
+ root_hash = build_merkle_tree('')[-1][0]
+ try:
+ logorder = get_logorder(logorderfile)
+ tree_size = len(logorder)
+ root_hash = build_merkle_tree(logorder[:tree_size])[-1][0]
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ pass
+ timestamp = int(time() * 1000)
+ tree_head_signature = \
+ generate_tree_head_signature(signingnodes, own_key,
+ tree_size, timestamp, root_hash)
+ if tree_head_signature == None:
+ return 1
+
+ sth = {"tree_size": tree_size,
+ "timestamp": timestamp,
+ "sha256_root_hash": b64encode(root_hash),
+ "tree_head_signature": b64encode(tree_head_signature)}
+ print "Creating sth file with tree size", tree_size, \
+ "and timestamp", timestamp
+ write_file(sthfile, sth)
+
return 0
if __name__ == '__main__':
diff --git a/tools/merge b/tools/merge
index d0198ec..19e22f8 100755
--- a/tools/merge
+++ b/tools/merge
@@ -5,6 +5,6 @@ set -o errexit
BINDIR=$(dirname $0)
$BINDIR/merge_fetch.py "$@"
-#$BINDIR/merge_backup.py "$@"
-$BINDIR/merge_sth.py "$@"
-#$BINDIR/merge_dist.py "$@"
+# $BINDIR/merge_backup.py "$@"
+# $BINDIR/merge_sth.py "$@"
+# $BINDIR/merge_dist.py "$@"