Test failing merge backup

Refactor tests

10 files changed, 92 insertions, 46 deletions

diff --git a/test/scripts/light-system-test-prepare-merge-takeover.sh b/test/scripts/light-system-test-prepare-merge-takeover.sh
index 5529805..c84a17e 100755
--- a/test/scripts/light-system-test-prepare-merge-takeover.sh
+++ b/test/scripts/light-system-test-prepare-merge-takeover.sh
@@ -5,7 +5,7 @@ set -o errexit
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
+. ${top_srcdir}/test/scripts/testutils.sh
 
 mv mergedb mergedb-down
 mv mergedb-secondary mergedb
diff --git a/test/scripts/light-system-test-prepare-redistribute-frontend.sh b/test/scripts/light-system-test-prepare-redistribute-frontend.sh
index 00d2070..cba93d4 100755
--- a/test/scripts/light-system-test-prepare-redistribute-frontend.sh
+++ b/test/scripts/light-system-test-prepare-redistribute-frontend.sh
@@ -5,7 +5,7 @@ set -o errexit
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
+. ${top_srcdir}/test/scripts/testutils.sh
 
 mv machine/machine-1 machine/machine-1-down
 mkdir -p machine/machine-1/db
diff --git a/test/scripts/light-system-test-prepare.sh b/test/scripts/light-system-test-prepare.sh
index 79785c1..466f3aa 100755
--- a/test/scripts/light-system-test-prepare.sh
+++ b/test/scripts/light-system-test-prepare.sh
@@ -7,7 +7,7 @@ SOFTHSM=/usr/local/bin/softhsm2-util
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
+. ${top_srcdir}/test/scripts/testutils.sh
 
 createca () {
     mkdir httpsca
diff --git a/test/scripts/light-system-test-run-1.sh b/test/scripts/light-system-test-run-1.sh
index bd55483..3b6266e 100755
--- a/test/scripts/light-system-test-run-1.sh
+++ b/test/scripts/light-system-test-run-1.sh
@@ -5,43 +5,53 @@ set -o errexit
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
+. ${top_srcdir}/test/scripts/testutils.sh
 
-python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test || (echo "Tests failed" ; false)
-${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem || (echo "Check failed" ; false)
-python ${top_srcdir}/tools/fetchallcerts.py ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (echo "Verification failed" ; false)
-python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || (echo "GC failed" ; false)
+python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test || fail "Tests failed"
+check_sth
+python ${top_srcdir}/tools/fetchallcerts.py ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification failed"
+python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || fail "GC failed"
 for certfile in ${top_srcdir}/tools/testcerts/cert[1-5].txt ${top_srcdir}/tools/testcerts/pre[12].txt; do
-    python ${top_srcdir}/tools/submitcert.py --parallel=1 --store $certfile --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (echo "Submission failed" ; false)
+    python ${top_srcdir}/tools/submitcert.py --parallel=1 --store $certfile --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 done
-python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || (echo "GC failed" ; false)
-${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || (echo "Merge failed" ; false)
-${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem || (echo "Check failed" ; false)
+python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || fail "GC failed"
+do_merge
+check_sth
+
+assert_equal "Tree size" "$(get_treesize)" 7
+
 mkdir fetchcertstore
-python ${top_srcdir}/tools/fetchallcerts.py ${BASEURL} --store fetchcertstore --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (echo "Verification failed" ; false)
+python ${top_srcdir}/tools/fetchallcerts.py ${BASEURL} --store fetchcertstore --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification failed"
 (cd fetchcertstore && unzip 0000.zip)
 
-python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert1.txt fetchcertstore/00000000 || (echo "Verification failed" ; false)
-python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert2.txt fetchcertstore/00000001 || (echo "Verification failed" ; false)
-python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert3.txt fetchcertstore/00000002 || (echo "Verification failed" ; false)
-python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert4.txt fetchcertstore/00000003 || (echo "Verification failed" ; false)
-python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert5.txt fetchcertstore/00000004 || (echo "Verification failed" ; false)
-python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/pre1.txt:${top_srcdir}/tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006 || (echo "Verification failed" ; false)
-python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg || (echo "GC failed" ; false)
+python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert1.txt fetchcertstore/00000000 || fail "Verification failed"
+python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert2.txt fetchcertstore/00000001 || fail "Verification failed"
+python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert3.txt fetchcertstore/00000002 || fail "Verification failed"
+python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert4.txt fetchcertstore/00000003 || fail "Verification failed"
+python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert5.txt fetchcertstore/00000004 || fail "Verification failed"
+python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/pre1.txt:${top_srcdir}/tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006 || fail"Verification failed"
+python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg || fail "GC failed"
 
 ${top_srcdir}/tools/to_catlfish.py to_erl nodes/merge-2/ "init:stop()"
-python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert6.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (echo "Submission failed" ; false)
+python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert6.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 echo NOTE: merge backup should fail with 111 Connection refused
-${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || (echo "Merge failed" ; false)
-treesize=$(${top_srcdir}/tools/loginfo.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg --treesize https://localhost:8080/) ;  test "${treesize}" = "7" || (echo "Tree size ${treesize} != expected 7" ; false)
-${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem
+
+${top_srcdir}/tools/merge_fetch.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed"
+${top_srcdir}/tools/merge_backup.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg && fail "Merge backup should have failed"
+${top_srcdir}/tools/merge_sth.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed"
+${top_srcdir}/tools/merge_dist.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed"
+
+assert_equal "Tree size" "$(get_treesize)" 7
+
+check_sth
 ../bin/run_erl -daemon nodes/merge-2/ nodes/merge-2/log/ "exec ../bin/erl -config merge-2"
 for i in 1 2 3 4 5 6 7 8 9 10; do
     echo "waiting for system to start" ; \
     sleep 0.5 ; \
     if curl -s --cacert httpsca/demoCA/cacert.pem -4 https://localhost:8181 > /dev/null ; then break; fi
 done
-${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || (echo "Merge failed" ; false)
-treesize=$(${top_srcdir}/tools/loginfo.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg --treesize https://localhost:8080/)
-test "${treesize}" = "8" || (echo "Tree size ${treesize} != expected 8" ; false)
-${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem
+do_merge
+
+assert_equal "Tree size" "$(get_treesize)" 8
+
+check_sth
diff --git a/test/scripts/light-system-test-run-2.sh b/test/scripts/light-system-test-run-2.sh
index 1a30c4c..4c8bbdb 100755
--- a/test/scripts/light-system-test-run-2.sh
+++ b/test/scripts/light-system-test-run-2.sh
@@ -5,8 +5,10 @@ set -o errexit
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
+. ${top_srcdir}/test/scripts/testutils.sh
 
-python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts --parallel 1 ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (echo "Verification of SCT:s failed" ; false)
-../../tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem
-python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || (echo "GC failed" ; false)
+python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts --parallel 1 ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification of SCT:s failed"
+
+check_sth
+
+python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || fail "GC failed"
diff --git a/test/scripts/light-system-test-run-3.sh b/test/scripts/light-system-test-run-3.sh
index f84317b..bb042d2 100755
--- a/test/scripts/light-system-test-run-3.sh
+++ b/test/scripts/light-system-test-run-3.sh
@@ -5,12 +5,22 @@ set -o errexit
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
-
-${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || (echo "Merge failed" ; false)
-${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem
-python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert7.txt --check-sct --sct-file=submittedcerts-7 ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (echo "Submission failed" ; false)
-${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || (echo "Merge failed" ; false)
-python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts --parallel 1 ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (echo "Verification of SCT:s failed"; false)
-python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts-7 --parallel 1 ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (echo "Verification of SCT:s failed"; false)
-${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem
+. ${top_srcdir}/test/scripts/testutils.sh
+
+do_merge
+check_sth
+
+assert_equal "Tree size" "$(get_treesize)" 8
+
+python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert7.txt --check-sct --sct-file=submittedcerts-7 ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
+
+assert_equal "Tree size" "$(get_treesize)" 8
+
+do_merge
+
+assert_equal "Tree size" "$(get_treesize)" 9
+
+python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts --parallel 1 ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification of SCT:s failed"
+python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts-7 --parallel 1 ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification of SCT:s failed"
+
+check_sth
diff --git a/test/scripts/light-system-test-run-4.sh b/test/scripts/light-system-test-run-4.sh
index 1022fd2..a2490f9 100755
--- a/test/scripts/light-system-test-run-4.sh
+++ b/test/scripts/light-system-test-run-4.sh
@@ -5,7 +5,9 @@ set -o errexit
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
+. ${top_srcdir}/test/scripts/testutils.sh
 
-${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || (echo "Merge failed" ; false)
-${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem
+do_merge
+check_sth
+
+assert_equal "Tree size" "$(get_treesize)" 9
diff --git a/test/scripts/light-system-test-start.sh b/test/scripts/light-system-test-start.sh
index 42cb4be..1623d7e 100755
--- a/test/scripts/light-system-test-start.sh
+++ b/test/scripts/light-system-test-start.sh
@@ -5,7 +5,7 @@ set -o errexit
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
+. ${top_srcdir}/test/scripts/testutils.sh
 
 for node in ${ERLANGNODES}; do
     (../bin/run_erl -daemon nodes/${node}/ nodes/${node}/log/ "exec ../bin/erl -config ${node}")
diff --git a/test/scripts/light-system-test-stop.sh b/test/scripts/light-system-test-stop.sh
index 1828e73..575ef38 100755
--- a/test/scripts/light-system-test-stop.sh
+++ b/test/scripts/light-system-test-stop.sh
@@ -5,7 +5,7 @@ set -o errexit
 
 top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
-. ${top_srcdir}/test/test.shvars
+. ${top_srcdir}/test/scripts/testutils.sh
 
 for node in ${ERLANGNODES}; do
     ${top_srcdir}/tools/to_catlfish.py to_erl nodes/${node}/ "init:stop()"
diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh
new file mode 100644
index 0000000..94d6223
--- /dev/null
+++ b/test/scripts/testutils.sh
@@ -0,0 +1,22 @@
+. ${top_srcdir}/test/test.shvars
+
+fail() {
+    echo "$@"
+    false
+}
+
+assert_equal() {
+    test "$2" = "$3" || fail "$1 $2 != expected $3"
+}
+
+get_treesize() {
+    ${top_srcdir}/tools/loginfo.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg --treesize https://localhost:8080/
+}
+
+check_sth() {
+    ${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem || fail "Check STH failed"
+}
+
+do_merge() {
+    ${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed"
+}