Fix bugs in old tests. Add tests for merge takeover.

submitcert.py was not exiting with error code when cert submission failed.
compileconfig.py was generating duplicate node names when Erlang nodes have several servers

3 files changed, 66 insertions, 10 deletions

diff --git a/Makefile b/Makefile
index 5435c43..d69e5ab 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,6 @@ tests-prepare:
 	mkdir $(INSTDIR)/tests/mergedb/chains
 	touch $(INSTDIR)/tests/mergedb/logorder
 	mkdir $(INSTDIR)/tests/mergedb-secondary
-	mkdir $(INSTDIR)/tests/mergedb-secondary/chains
 	touch $(INSTDIR)/tests/mergedb-secondary/logorder
 	printf 0 > $(INSTDIR)/tests/mergedb-secondary/verifiedsize
 	mkdir $(INSTDIR)/tests/known_roots
@@ -76,7 +75,7 @@ tests-start:
 
 tests-run:
 	@(cd $(INSTDIR) && python ../tools/testcase1.py https://localhost:8080/ tests/keys/logkey.pem tests/httpsca/demoCA/cacert.pem) || (echo "Tests failed" ; false)
-	@diff -r -x nursery -x verifiedsize -x fetched -x sth -x verified.\* catlfish/tests/mergedb catlfish/tests/mergedb-secondary || (echo "Merge databases not matching" ; false)
+	@(cd $(INSTDIR) && ../tools/verifysth.py $(BASEURL) --publickey=tests/keys/logkey.pem --cur-sth=stored-sth --cafile tests/httpsca/demoCA/cacert.pem)
 	@(cd $(INSTDIR) && python ../tools/check-sth.py --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem https://localhost:8080/) || (echo "Check failed" ; false)
 	@(cd $(INSTDIR) && python ../tools/fetchallcerts.py $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Verification failed" ; false)
 	@(cd $(INSTDIR) && rm -f submittedcerts)
@@ -90,7 +89,7 @@ tests-run:
 	@(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre2.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false)
 	@(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg)  || (echo "GC failed" ; false)
 	@(cd $(INSTDIR) && ../tools/merge --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
-	@diff -r -x nursery -x verifiedsize -x fetched -x sth -x verified.\* catlfish/tests/mergedb catlfish/tests/mergedb-secondary || (echo "Merge databases not matching" ; false)
+	@(cd $(INSTDIR) && ../tools/verifysth.py $(BASEURL) --publickey=tests/keys/logkey.pem --cur-sth=stored-sth --cafile tests/httpsca/demoCA/cacert.pem)
 	@(cd $(INSTDIR) && python ../tools/check-sth.py --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem https://localhost:8080/) || (echo "Check failed" ; false)
 	@(cd $(INSTDIR) && rm -r fetchcertstore || true)
 	@(cd $(INSTDIR) && mkdir fetchcertstore)
@@ -110,6 +109,7 @@ tests-run:
 	@(cd $(INSTDIR) && ../tools/merge --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
 	@(cd $(INSTDIR) && treesize=$$(../tools/loginfo.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg --treesize https://localhost:8080/) ; \
 	  test "$$treesize" = "7" || (echo "Tree size $$treesize != expected 7" ; false))
+	@(cd $(INSTDIR) && ../tools/verifysth.py $(BASEURL) --publickey=tests/keys/logkey.pem --cur-sth=stored-sth --cafile tests/httpsca/demoCA/cacert.pem)
 	@(cd $(INSTDIR) && bin/run_erl -daemon ../test/nodes/merge-2/ ../test/nodes/merge-2/log/ "exec bin/erl -config merge-2")
 	@for i in 1 2 3 4 5 6 7 8 9 10; do \
 	  echo "waiting for system to start" ; \
@@ -119,8 +119,42 @@ tests-run:
 	@(cd $(INSTDIR) && ../tools/merge --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
 	@(cd $(INSTDIR) && treesize=$$(../tools/loginfo.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg --treesize https://localhost:8080/) ; \
 	  test "$$treesize" = "8" || (echo "Tree size $$treesize != expected 8" ; false))
+	@(cd $(INSTDIR) && ../tools/verifysth.py $(BASEURL) --publickey=tests/keys/logkey.pem --cur-sth=stored-sth --cafile tests/httpsca/demoCA/cacert.pem)
+
 tests-run2:
 	@(cd $(INSTDIR) ; python ../tools/verifysct.py --sct-file=submittedcerts --parallel 1 $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || echo "Verification of SCT:s failed"
+	@(cd $(INSTDIR) && ../tools/verifysth.py $(BASEURL) --publickey=tests/keys/logkey.pem --cur-sth=stored-sth --cafile tests/httpsca/demoCA/cacert.pem)
+	@(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg)  || (echo "GC failed" ; false)
+
+tests-prepare-merge-takeover:
+	@echo $@
+	mv $(INSTDIR)/tests/mergedb catlfish/tests/mergedb-down
+	mv $(INSTDIR)/tests/mergedb-secondary catlfish/tests/mergedb
+	mkdir $(INSTDIR)/tests/mergedb-secondary
+	touch $(INSTDIR)/tests/mergedb-secondary/logorder
+	printf 0 > $(INSTDIR)/tests/mergedb-secondary/verifiedsize
+
+tests-run3:
+	@echo $@
+	@(cd $(INSTDIR) && ../tools/merge --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
+	@(cd $(INSTDIR) && ../tools/verifysth.py $(BASEURL) --publickey=tests/keys/logkey.pem --cur-sth=stored-sth --cafile tests/httpsca/demoCA/cacert.pem)
+	@(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert7.txt --check-sct --sct-file=submittedcerts-7 $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false)
+	@(cd $(INSTDIR) && ../tools/merge --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
+	@(cd $(INSTDIR) ; python ../tools/verifysct.py --sct-file=submittedcerts --parallel 1 $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || echo "Verification of SCT:s failed"
+	@(cd $(INSTDIR) && ../tools/verifysth.py $(BASEURL) --publickey=tests/keys/logkey.pem --cur-sth=stored-sth --cafile tests/httpsca/demoCA/cacert.pem)
+
+
+tests-prepare-redistribute-frontend:
+	@echo $@
+	mv $(INSTDIR)/tests/machine/machine-1 $(INSTDIR)/tests/machine/machine-1-down && \
+	mkdir -p $(INSTDIR)/tests/machine/machine-1/db && \
+	touch $(INSTDIR)/tests/machine/machine-1/db/index && \
+	touch $(INSTDIR)/tests/machine/machine-1/db/newentries ; \
+
+tests-run4:
+	@echo $@
+	@(cd $(INSTDIR) && ../tools/merge --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
+	@(cd $(INSTDIR) && ../tools/verifysth.py $(BASEURL) --publickey=tests/keys/logkey.pem --cur-sth=stored-sth --cafile tests/httpsca/demoCA/cacert.pem)
 
 tests-stop:
 	@for node in $(NODES); do \
@@ -188,6 +222,18 @@ tests:
 	@make tests-run2 || (sleep 5; make tests-stop ; false)
 	@make tests-wait
 	@make tests-stop
+	@make tests-wait
+	@make tests-prepare-merge-takeover
+	@make tests-start
+	@make tests-run3 || (sleep 5; make tests-stop ; false)
+	@make tests-wait
+	@make tests-stop
+	@make tests-wait
+	@make tests-prepare-redistribute-frontend
+	@make tests-start
+	@make tests-run4 || (sleep 5; make tests-stop ; false)
+	@make tests-wait
+	@make tests-stop
 
 tests-formatupgrade:
 	@make tests-makemk
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 777a12f..d9ac740 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -350,19 +350,19 @@ def gen_config(nodename, config, localconfig):
 
 def gen_testmakefile(config, testmakefile, machines):
     configfile = open(testmakefile, "w")
-    frontendnodenames = [node["name"] for node in config["frontendnodes"]]
-    storagenodenames = [node["name"] for node in config["storagenodes"]]
-    signingnodenames = [node["name"] for node in config["signingnodes"]]
-    mergenodenames = [node["name"] for node in config["mergenodes"]]
-    erlangnodenames = frontendnodenames + storagenodenames + signingnodenames + \
-      filter(lambda name: name != config["primarymergenode"], mergenodenames)
+    frontendnodenames = set([node["name"] for node in config["frontendnodes"]])
+    storagenodenames = set([node["name"] for node in config["storagenodes"]])
+    signingnodenames = set([node["name"] for node in config["signingnodes"]])
+    mergenodenames = set([node["name"] for node in config["mergenodes"]])
+    erlangnodenames = frontendnodenames | storagenodenames | signingnodenames | \
+      set(filter(lambda name: name != config["primarymergenode"], mergenodenames))
 
     frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]]
     storagenodeaddresses = [node["address"] for node in config["storagenodes"]]
     signingnodeaddresses = [node["address"] for node in config["signingnodes"]]
     mergenodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]
 
-    print >>configfile, "NODES=" + " ".join(frontendnodenames+storagenodenames+signingnodenames+mergenodenames)
+    print >>configfile, "NODES=" + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames)
     print >>configfile, "ERLANGNODES=" + " ".join(erlangnodenames)
     print >>configfile, "MACHINES=" + " ".join([str(e) for e in range(1, machines+1)])
     print >>configfile, "TESTURLS=" + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergenodeaddresses)
diff --git a/tools/submitcert.py b/tools/submitcert.py
index 3b14912..66d5126 100755
--- a/tools/submitcert.py
+++ b/tools/submitcert.py
@@ -183,11 +183,15 @@ print ncerts, "certs"
 
 certs = get_all_certificates(certfiles)
 
+errors = 0
+
 (result, timing) = submitcert(certs.next())
 if result != None:
     nsubmitted += 1
     (leafcert, issuer_key_hash, sct) = result
     save_sct(sct, sth, leafcert, issuer_key_hash)
+else:
+    errors += 1
 
 if args.pre_warm:
     select.select([], [], [], 3.0)
@@ -206,6 +210,8 @@ try:
             nsubmitted += 1
             (leafcert, issuer_key_hash, sct) = result
             save_sct(sct, sth, leafcert, issuer_key_hash)
+        else:
+            errors += 1
         deltatime = datetime.datetime.now() - starttime
         deltatime_f = deltatime.seconds + deltatime.microseconds / 1000000.0
         rate = nsubmitted / deltatime_f
@@ -217,3 +223,7 @@ try:
 except KeyboardInterrupt:
     p.terminate()
     p.join()
+
+if errors:
+    print >> sys.stderr, errors, "errors encountered, exiting with error code"
+    sys.exit(1)