Refactor tests

24 files changed, 325 insertions, 133 deletions

diff --git a/test/Makefile b/test/Makefile
index a033de6..1074348 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -11,7 +11,7 @@ tests-wait:
 	sleep 5
 
 tests-makemk:
-	$(PREFIX)/tools/compileconfig.py --config=$(PREFIX)/test/catlfish-test.cfg.in --testshellvars=$(PREFIX)/test/test.shvars --machines 1
+	$(PREFIX)/tools/compileconfig.py --config=$(PREFIX)/test/catlfish-test.cfg.in --testshellvars=$(PREFIX)/test/test.shvars
 
 tests:
 	@make tests-makemk
diff --git a/test/catlfish-test-local-1.cfg b/test/catlfish-test-local-frontend-1.cfg
index 21f8c17..4867672 100644
--- a/test/catlfish-test-local-1.cfg
+++ b/test/catlfish-test-local-frontend-1.cfg
@@ -21,13 +21,13 @@ paths:
         https_keyfile: httpscert/httpskey-1.pem
         https_cacertfile: httpsca/demoCA/cacert.pem
         public_cacertfile: httpsca/demoCA/cacert.pem
-        db: machine/machine-1/db/
+        db: machine/frontend-1/db/
         publickeys: publickeys
         logpublickey: keys/logkey.pem
         privatekeys: privatekeys
 
 ratelimits:
-        add_chain: 10 per second
+        add_chain: 1000 per second
 
 logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
     Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
diff --git a/test/catlfish-test-local-frontend-2.cfg b/test/catlfish-test-local-frontend-2.cfg
new file mode 100644
index 0000000..2a6462a
--- /dev/null
+++ b/test/catlfish-test-local-frontend-2.cfg
@@ -0,0 +1,23 @@
+localnodes:
+        - frontend-2
+
+paths:
+        configdir: .
+        knownroots: known_roots
+        https_certfile: httpscert/httpscert-1.pem
+        https_keyfile: httpscert/httpskey-1.pem
+        https_cacertfile: httpsca/demoCA/cacert.pem
+        public_cacertfile: httpsca/demoCA/cacert.pem
+        db: machine/frontend-2/db/
+        publickeys: publickeys
+        logpublickey: keys/logkey.pem
+        privatekeys: privatekeys
+
+ratelimits:
+        add_chain: 1000 per second
+
+logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
+    Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
+    0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-frontend-3.cfg b/test/catlfish-test-local-frontend-3.cfg
new file mode 100644
index 0000000..430cef6
--- /dev/null
+++ b/test/catlfish-test-local-frontend-3.cfg
@@ -0,0 +1,23 @@
+localnodes:
+        - frontend-3
+
+paths:
+        configdir: .
+        knownroots: known_roots
+        https_certfile: httpscert/httpscert-1.pem
+        https_keyfile: httpscert/httpskey-1.pem
+        https_cacertfile: httpsca/demoCA/cacert.pem
+        public_cacertfile: httpsca/demoCA/cacert.pem
+        db: machine/frontend-3/db/
+        publickeys: publickeys
+        logpublickey: keys/logkey.pem
+        privatekeys: privatekeys
+
+ratelimits:
+        add_chain: 1000 per second
+
+logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
+    Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
+    0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge-1.cfg
index 9283136..389a440 100644
--- a/test/catlfish-test-local-merge.cfg
+++ b/test/catlfish-test-local-merge-1.cfg
@@ -5,7 +5,7 @@ localnodes:
 
 paths:
         configdir: .
-        mergedb: mergedb
+        mergedb: machine/merge-1/db
         # TODO: https_certfile and https_keyfile not strictly necessary for primary merge
         https_certfile: httpscert/httpscert-1.pem
         https_keyfile: httpscert/httpskey-1.pem
diff --git a/test/catlfish-test-local-merge-2.cfg b/test/catlfish-test-local-merge-2.cfg
index e2baa01..d42cbf3 100644
--- a/test/catlfish-test-local-merge-2.cfg
+++ b/test/catlfish-test-local-merge-2.cfg
@@ -2,7 +2,7 @@ localnodes:
         - merge-2
 
 addresses:
-        merge-2: 127.0.0.1:8181
+        merge-2: 127.0.0.1:8182
 
 nodename: merge-2
 
@@ -11,16 +11,26 @@ nodename: merge-2
 paths:
         configdir: .
         knownroots: known_roots
-        mergedb: mergedb-secondary
+        mergedb: machine/merge-2/db
         https_certfile: httpscert/httpscert-1.pem
         https_keyfile: httpscert/httpskey-1.pem
         https_cacertfile: httpsca/demoCA/cacert.pem
         publickeys: publickeys
         logpublickey: keys/logkey.pem
         privatekeys: privatekeys
+        verifycert_bin: ../bin/verifycert.erl.escript
 
 logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
     Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
     0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
 
+merge:
+        min-delay: 1
+        backup-window-size: 2
+        backup-sendlog-chunksize: 1
+        backup-sendentries-chunksize: 1
+        dist-window-size: 2
+        dist-sendlog-chunksize: 1
+        dist-sendentries-chunksize: 1
+
 configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-merge-3.cfg b/test/catlfish-test-local-merge-3.cfg
new file mode 100644
index 0000000..f6da89e
--- /dev/null
+++ b/test/catlfish-test-local-merge-3.cfg
@@ -0,0 +1,24 @@
+localnodes:
+        - merge-3
+
+addresses:
+        merge-3: 127.0.0.1:8183
+
+nodename: merge-3
+
+paths:
+        configdir: .
+        knownroots: known_roots
+        mergedb: machine/merge-3/db
+        https_certfile: httpscert/httpscert-1.pem
+        https_keyfile: httpscert/httpskey-1.pem
+        https_cacertfile: httpsca/demoCA/cacert.pem
+        publickeys: publickeys
+        logpublickey: keys/logkey.pem
+        privatekeys: privatekeys
+
+logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
+    Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
+    0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-signing.cfg b/test/catlfish-test-local-signing-1.cfg
index 0eccc94..0eccc94 100644
--- a/test/catlfish-test-local-signing.cfg
+++ b/test/catlfish-test-local-signing-1.cfg
diff --git a/test/catlfish-test-mergefailover.cfg.in b/test/catlfish-test-mergefailover.cfg.in
new file mode 100644
index 0000000..2e2cd1a
--- /dev/null
+++ b/test/catlfish-test-mergefailover.cfg.in
@@ -0,0 +1,40 @@
+baseurl: https://localhost:8080/
+
+frontendnodes:
+        - name: frontend-1
+          publicaddress: localhost:8080
+          address: localhost:8082
+        - name: frontend-2
+          publicaddress: localhost:8090
+          address: localhost:8092
+        - name: frontend-3
+          publicaddress: localhost:8000
+          address: localhost:8002
+
+storagenodes:
+        - name: frontend-1
+          address: localhost:8081
+        - name: frontend-2
+          address: localhost:8091
+        - name: frontend-3
+          address: localhost:8001
+
+signingnodes:
+        - name: signing-1
+          address: localhost:8088
+
+mergenodes:
+        - name: merge-1
+          address: localhost:8181
+        - name: merge-2
+          address: localhost:8182
+        - name: merge-3
+          address: localhost:8183
+
+primarymergenode: merge-2
+
+backup-quorum-size: 1
+
+storage-quorum-size: 1
+
+mmd: 86400
diff --git a/test/catlfish-test-redistribute-frontend.cfg.in b/test/catlfish-test-redistribute-frontend.cfg.in
new file mode 100644
index 0000000..725c7c5
--- /dev/null
+++ b/test/catlfish-test-redistribute-frontend.cfg.in
@@ -0,0 +1,45 @@
+baseurl: https://localhost:8000/
+
+frontendnodes:
+        - name: frontend-1
+          publicaddress: localhost:8080
+          address: localhost:8082
+        - name: frontend-2
+          publicaddress: localhost:8090
+          address: localhost:8092
+        - name: frontend-3
+          publicaddress: localhost:8000
+          address: localhost:8002
+
+storagenodes:
+        - name: frontend-1
+          address: localhost:8081
+        - name: frontend-2
+          address: localhost:8091
+        - name: frontend-3
+          address: localhost:8001
+
+signingnodes:
+        - name: signing-1
+          address: localhost:8088
+
+mergenodes:
+        - name: merge-1
+          address: localhost:8181
+        - name: merge-2
+          address: localhost:8182
+        - name: merge-3
+          address: localhost:8183
+
+primarymergenode: merge-2
+
+backup-quorum-size: 1
+
+storage-quorum-size: 1
+
+mmd: 86400
+
+statsservers:
+        - name: statsserver
+          address: localhost:9081
+          publicaddress: localhost:9082
diff --git a/test/catlfish-test.cfg.in b/test/catlfish-test.cfg.in
index 68c753f..c3b59b0 100644
--- a/test/catlfish-test.cfg.in
+++ b/test/catlfish-test.cfg.in
@@ -4,10 +4,20 @@ frontendnodes:
         - name: frontend-1
           publicaddress: localhost:8080
           address: localhost:8082
+        - name: frontend-2
+          publicaddress: localhost:8090
+          address: localhost:8092
+        - name: frontend-3
+          publicaddress: localhost:8000
+          address: localhost:8002
 
 storagenodes:
         - name: frontend-1
           address: localhost:8081
+        - name: frontend-2
+          address: localhost:8091
+        - name: frontend-3
+          address: localhost:8001
 
 signingnodes:
         - name: signing-1
@@ -15,8 +25,11 @@ signingnodes:
 
 mergenodes:
         - name: merge-1
-        - name: merge-2
           address: localhost:8181
+        - name: merge-2
+          address: localhost:8182
+        - name: merge-3
+          address: localhost:8183
 
 primarymergenode: merge-1
 
diff --git a/test/scripts/light-system-test-prepare-merge-takeover.sh b/test/scripts/light-system-test-prepare-merge-takeover.sh
index 057af3d..f27b7f9 100755
--- a/test/scripts/light-system-test-prepare-merge-takeover.sh
+++ b/test/scripts/light-system-test-prepare-merge-takeover.sh
@@ -7,9 +7,8 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 . ${top_srcdir}/test/scripts/testutils.sh
 
-mv mergedb mergedb-down
-mv mergedb-secondary mergedb
-mv mergedb/verifiedsize mergedb/verifiedsize.OFF
-mkdir mergedb-secondary
-touch mergedb-secondary/logorder
-printf 0 > mergedb-secondary/verifiedsize
+mv machine/merge-1/db/sth machine/merge-1/db/sth.OFF
+
+${top_srcdir}/tools/initlog.py --config machine/merge-2/catlfish-test.cfg --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg --promote-secondary
+
+cp machine/merge-2/catlfish-test-local-merge-2.cfg current-merge-localconfig.cfg
diff --git a/test/scripts/light-system-test-prepare-redistribute-frontend.sh b/test/scripts/light-system-test-prepare-redistribute-frontend.sh
deleted file mode 100755
index c7b6f82..0000000
--- a/test/scripts/light-system-test-prepare-redistribute-frontend.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-
-set -o nounset
-set -o errexit
-
-top_srcdir=$(cd $(dirname $0)/../..; pwd)
-
-. ${top_srcdir}/test/scripts/testutils.sh
-
-mv machine/machine-1 machine/machine-1-down
-mkdir -p machine/machine-1/db
-touch machine/machine-1/db/index
-touch machine/machine-1/db/newentries
-cp machine/machine-1-down/catlfish-test-local-1.cfg machine/machine-1/
-cp machine/machine-1-down/catlfish-test.cfg machine/machine-1/
-cp machine/machine-1-down/catlfish-test.cfg.sig machine/machine-1/
diff --git a/test/scripts/light-system-test-prepare.sh b/test/scripts/light-system-test-prepare.sh
index 1f68ed1..9b500c9 100755
--- a/test/scripts/light-system-test-prepare.sh
+++ b/test/scripts/light-system-test-prepare.sh
@@ -41,39 +41,29 @@ createcert
 mkdir keys
 (cd keys ; ${top_srcdir}/tools/create-key.sh logkey)
 openssl pkcs8 -topk8 -nocrypt -in keys/logkey-private.pem -out keys/logkey-private.pkcs8
-mkdir mergedb
-mkdir mergedb-secondary
-touch mergedb-secondary/logorder
-printf 0 > mergedb-secondary/verifiedsize
+
 mkdir known_roots
 cp ${top_srcdir}/tools/testcerts/roots/* known_roots
 mkdir privatekeys
 
-for node in ${NODES}; do \
+for node in ${NODES}; do
     (cd privatekeys ; ${top_srcdir}/tools/create-key.sh ${node})
     mkdir -p nodes/${node}/log
+    mkdir -p machine/${node}/db
+    cp ${top_srcdir}/test/catlfish-test-local-${node}.cfg machine/${node}/
 done
 
-${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
-
-for machine in ${MACHINES}; do \
-    mkdir -p machine/machine-${machine}/db
-    cp ${top_srcdir}/test/catlfish-test-local-${machine}.cfg machine/machine-${machine}/
-    ${top_srcdir}/tools/getconfig.py --localconfig machine/machine-${machine}/catlfish-test-local-${machine}.cfg --dest machine/machine-${machine}/catlfish-test.cfg
-    ${top_srcdir}/tools/compileconfig.py --config machine/machine-${machine}/catlfish-test.cfg --localconfig machine/machine-${machine}/catlfish-test-local-${machine}.cfg
-    touch machine/machine-${machine}/db/index && touch machine/machine-${machine}/db/newentries
+for node in ${MERGESECONDARYNODES}; do
+    nodename=$(echo ${node} | cut -f 1 -d ':')
+    printf 0 > machine/${nodename}/db/verifiedsize
 done
-mkdir -p machine/merge
-cp ${top_srcdir}/test/catlfish-test-local-merge.cfg machine/merge/
-${top_srcdir}/tools/getconfig.py --localconfig machine/merge/catlfish-test-local-merge.cfg --dest machine/merge/catlfish-test.cfg
-${top_srcdir}/tools/compileconfig.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
-mkdir -p machine/merge-2
-cp ${top_srcdir}/test/catlfish-test-local-merge-2.cfg machine/merge-2/
-${top_srcdir}/tools/getconfig.py --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg --dest machine/merge-2/catlfish-test.cfg
-${top_srcdir}/tools/compileconfig.py --config machine/merge-2/catlfish-test.cfg --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg
-mkdir -p machine/signing
-cp ${top_srcdir}/test/catlfish-test-local-signing.cfg machine/signing/
-${top_srcdir}/tools/getconfig.py --localconfig machine/signing/catlfish-test-local-signing.cfg --dest machine/signing/catlfish-test.cfg
-${top_srcdir}/tools/compileconfig.py --config machine/signing/catlfish-test.cfg --localconfig machine/signing/catlfish-test-local-signing.cfg
+
+
 test -x ${SOFTHSM} && ${SOFTHSM} --init-token --slot=0 --label=mylabel --so-pin=ffff --pin=ffff || true
 test -x ${SOFTHSM} && ${SOFTHSM} --import keys/logkey-private.pkcs8 --slot 0 --label mylabel --pin ffff --id 00 || true
+
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+
+${top_srcdir}/test/scripts/reload-config.sh
+
+cp machine/merge-1/catlfish-test-local-merge-1.cfg current-merge-localconfig.cfg
diff --git a/test/scripts/light-system-test-run-1.sh b/test/scripts/light-system-test-run-1.sh
index 7d52ed7..0695497 100755
--- a/test/scripts/light-system-test-run-1.sh
+++ b/test/scripts/light-system-test-run-1.sh
@@ -7,14 +7,14 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 . ${top_srcdir}/test/scripts/testutils.sh
 
-python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem machine/merge || fail "Tests failed"
+python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem machine/merge-1 || fail "Tests failed"
 check_sth
 python ${top_srcdir}/tools/fetchallcerts.py ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification failed"
-python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg  || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/frontend-1/catlfish-test.cfg --localconfig machine/frontend-1/catlfish-test-local-frontend-1.cfg  || fail "GC failed"
 for certfile in ${top_srcdir}/tools/testcerts/cert[1-5].txt ${top_srcdir}/tools/testcerts/pre[12].txt; do
     python ${top_srcdir}/tools/submitcert.py --parallel=1 --store $certfile --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 done
-python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg  || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/frontend-1/catlfish-test.cfg --localconfig machine/frontend-1/catlfish-test-local-frontend-1.cfg  || fail "GC failed"
 do_merge 7
 check_sth
 
@@ -30,9 +30,9 @@ python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert3.tx
 python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert4.txt fetchcertstore/00000003 || fail "Verification failed"
 python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert5.txt fetchcertstore/00000004 || fail "Verification failed"
 python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/pre1.txt:${top_srcdir}/tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006 || fail"Verification failed"
-python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/frontend-1/catlfish-test.cfg --localconfig machine/frontend-1/catlfish-test-local-frontend-1.cfg || fail "GC failed"
 
-${top_srcdir}/test/scripts/light-system-test-stop.sh mergesecondary
+${top_srcdir}/test/scripts/light-system-test-stop.sh merge_2only
 
 python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert6.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 echo NOTE: merge backup should fail with 111 Connection refused
@@ -45,7 +45,7 @@ assert_equal "Tree size" "$(get_treesize)" 7
 check_sth
 
 
-${top_srcdir}/test/scripts/light-system-test-start.sh mergesecondary
+${top_srcdir}/test/scripts/light-system-test-start.sh merge_2only
 
 do_merge 8
 
diff --git a/test/scripts/light-system-test-run-2.sh b/test/scripts/light-system-test-run-2.sh
index bf80c97..88df7c1 100755
--- a/test/scripts/light-system-test-run-2.sh
+++ b/test/scripts/light-system-test-run-2.sh
@@ -11,4 +11,4 @@ python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts --parallel 1 $
 
 check_sth
 
-python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg  || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/frontend-1/catlfish-test.cfg --localconfig machine/frontend-1/catlfish-test-local-frontend-1.cfg  || fail "GC failed"
diff --git a/test/scripts/light-system-test-run-5.sh b/test/scripts/light-system-test-run-5.sh
index cf0bfaf..73e8f5f 100755
--- a/test/scripts/light-system-test-run-5.sh
+++ b/test/scripts/light-system-test-run-5.sh
@@ -7,12 +7,7 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 . ${top_srcdir}/test/scripts/testutils.sh
 
-${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
-
-for machine in ${MACHINES}; do \
-    ${top_srcdir}/tools/getconfig.sh machine/machine-${machine}/catlfish-test-local-${machine}.cfg machine/machine-${machine}/catlfish-test.cfg frontend-${machine}.plopcontrol
-done
-
+${top_srcdir}/test/scripts/reload-config.sh
 
 do_merge 9
 sleep 3
diff --git a/test/scripts/light-system-test-stop.sh b/test/scripts/light-system-test-stop.sh
index a946079..aa379cd 100755
--- a/test/scripts/light-system-test-stop.sh
+++ b/test/scripts/light-system-test-stop.sh
@@ -12,6 +12,7 @@ nodetypes="$@"
 stop_node() {
     node=$1
     ${top_srcdir}/tools/to_catlfish.py to_erl nodes/${node}/ "init:stop()"
+    rm ${node}.plopcontrol || true
 }
 
 nodes=
diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh
index 217fb2c..69eb5c7 100755
--- a/test/scripts/light-system-test.sh
+++ b/test/scripts/light-system-test.sh
@@ -18,16 +18,18 @@ tests_stop() {
 }
 
 tests_stop_all() {
-    ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend storage signing
+    ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend signing
 }
 
 ${SCRIPTS}/light-system-test-prepare.sh
-tests_start signing mergesecondary storage frontend
-${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+tests_start signing
+tests_start merge_2only frontendexceptlast
+${top_srcdir}/tools/initlog.py --config machine/merge-1/catlfish-test.cfg --localconfig machine/merge-1/catlfish-test-local-merge-1.cfg
 tests_start mergeprimary
 ${SCRIPTS}/light-system-test-run-1.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
+
 sleep 5
-tests_stop_all
+tests_stop mergeprimary merge_2only frontendexceptlast
 sleep 5
 echo
 echo
@@ -39,10 +41,10 @@ echo
 echo
 echo
 echo
-tests_start signing mergesecondary storage frontend mergeprimary
+tests_start merge_2only frontendexceptlast mergeprimary
 ${SCRIPTS}/light-system-test-run-2.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 sleep 5
-tests_stop_all
+tests_stop mergeprimary merge_2only
 sleep 5
 echo
 echo
@@ -54,13 +56,13 @@ echo
 echo
 echo
 echo
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-mergefailover.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+${top_srcdir}/test/scripts/reload-config.sh
 ${SCRIPTS}/light-system-test-prepare-merge-takeover.sh
-tests_start signing mergesecondary storage frontend
-${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg --promote-secondary
-tests_start mergeprimary
+tests_start merge_3only mergeprimary
 ${SCRIPTS}/light-system-test-run-3.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 sleep 5
-tests_stop_all
+tests_stop mergeprimary merge_3only frontendexceptlast
 sleep 5
 echo
 echo
@@ -72,8 +74,9 @@ echo
 echo
 echo
 echo
-${SCRIPTS}/light-system-test-prepare-redistribute-frontend.sh
-tests_start signing mergesecondary storage frontend mergeprimary
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-redistribute-frontend.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+${top_srcdir}/test/scripts/reload-config.sh
+tests_start merge_3only frontendexceptfirst mergeprimary
 ${SCRIPTS}/light-system-test-run-4.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 
 echo
@@ -87,6 +90,7 @@ echo
 echo
 echo
 
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-redistribute-frontend.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
 
 ${SCRIPTS}/light-system-test-run-5.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 sleep 5
diff --git a/test/scripts/perf-test.sh b/test/scripts/perf-test.sh
index 3766cf1..51e719c 100755
--- a/test/scripts/perf-test.sh
+++ b/test/scripts/perf-test.sh
@@ -18,66 +18,73 @@ tests_stop() {
 }
 
 tests_stop_all() {
-    ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend storage signing
+    ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend signing
 }
 
 ${SCRIPTS}/light-system-test-prepare.sh
-${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+${top_srcdir}/tools/initlog.py --config machine/merge-1/catlfish-test.cfg --localconfig machine/merge-1/catlfish-test-local-merge-1.cfg
 
 cp ${top_srcdir}/test/known_roots/* known_roots
 
 tests_start signing
-tests_start mergesecondary storage frontend mergeprimary
+tests_start merge_2only frontendexceptlast mergeprimary
 
 do_merge 0
 check_sth
 
 assert_equal "Tree size" "$(get_treesize)" 0
 
-python ${top_srcdir}/tools/submitcert.py --parallel=30 --store ${top_srcdir}/test/bulktestcerts/0000.zip --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (tests_stop ; fail "Submission failed")
+python ${top_srcdir}/tools/submitcert.py --parallel=30 --store ${top_srcdir}/test/bulktestcerts/0000.zip --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (tests_stop ; fail "Submission failed") &
 
 do_merge 10000 || (tests_stop ; fail "Merge failed")
 check_sth || (tests_stop ; fail "Check STH failed")
 
-wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg" "10000 10000 10000" 400
+python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg  || fail "GC failed" &
+python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-2/catlfish-test-local-2.cfg  || fail "GC failed" &
+
+wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge-1/catlfish-test.cfg --localconfig current-merge-localconfig.cfg" "10000 10000 10000" 400
+
 
 sleep 5
-tests_stop mergesecondary storage frontend mergeprimary
+tests_stop merge_2only mergeprimary
 sleep 5
 
-mv mergedb mergedb-down
-mv mergedb-secondary mergedb
-mkdir mergedb-secondary
-touch mergedb-secondary/logorder
-printf 0 > mergedb-secondary/verifiedsize
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-mergefailover.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+${top_srcdir}/test/scripts/reload-config.sh
+${SCRIPTS}/light-system-test-prepare-merge-takeover.sh
 
-${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg --promote-secondary
+tests_start mergeprimary merge_3only
 
-tests_start mergesecondary storage frontend mergeprimary
+python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert1.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 
-do_merge 10000 || (tests_stop ; fail "Merge failed")
+do_merge 10001 || (tests_stop ; fail "Merge failed")
 check_sth || (tests_stop ; fail "Check STH failed")
 
-wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg" "10000 10000 10000" 400
+
+wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge-1/catlfish-test.cfg --localconfig current-merge-localconfig.cfg" "10001 10001 10001" 400
+
 
 sleep 5
-tests_stop mergesecondary storage frontend mergeprimary
+tests_stop mergesecondary frontendexceptlast mergeprimary
 sleep 5
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-redistribute-frontend.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+${top_srcdir}/test/scripts/reload-config.sh
 
-mv machine/machine-1 machine/machine-1-down
-mkdir -p machine/machine-1/db
-touch machine/machine-1/db/index
-touch machine/machine-1/db/newentries
+. ${top_srcdir}/test/test.shvars
 
-tests_start mergesecondary storage frontend mergeprimary
+tests_start mergesecondary frontendexceptfirst mergeprimary
 
-do_merge 10000 || (tests_stop ; fail "Merge failed")
-wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg" "10000 10000 10000" 400
+python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert2.txt --check-sct --sct-file=submittedcerts https://localhost:8090/ --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
+
+do_merge 10002 || (tests_stop ; fail "Merge failed")
+wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge-1/catlfish-test.cfg --localconfig current-merge-localconfig.cfg" "10002 10002 10002" 800
 check_sth || (tests_stop ; fail "Check STH failed")
 
 sleep 5
 tests_stop_all
 sleep 5
 
+echo Success
+
 #grep timing: bench-[123] > bench.txt
 #${top_srcdir}/tools/parsebench.py bench.txt > bench.html
diff --git a/test/scripts/reload-config.sh b/test/scripts/reload-config.sh
new file mode 100755
index 0000000..7dbeebe
--- /dev/null
+++ b/test/scripts/reload-config.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -o nounset
+set -o errexit
+
+top_srcdir=$(cd $(dirname $0)/../..; pwd)
+
+. ${top_srcdir}/test/scripts/testutils.sh
+
+${top_srcdir}/tools/compileconfig.py --config=catlfish-test-generated.cfg --testshellvars=${top_srcdir}/test/test.shvars
+
+for node in ${NODES}; do
+    ${top_srcdir}/tools/getconfig.sh machine/${node}/catlfish-test-local-${node}.cfg machine/${node}/catlfish-test.cfg ${node}.plopcontrol
+done
diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh
index 88e9a54..4d49be8 100644
--- a/test/scripts/testutils.sh
+++ b/test/scripts/testutils.sh
@@ -25,7 +25,7 @@ get_treesize() {
     get_treesize_host ${BASEURL}
 }
 get_treesize_host() {
-    ${top_srcdir}/tools/loginfo.py --localconfig machine/merge/catlfish-test-local-merge.cfg --treesize "$1"
+    ${top_srcdir}/tools/loginfo.py --localconfig current-merge-localconfig.cfg --treesize "$1"
 }
 
 check_sth() {
@@ -36,11 +36,11 @@ check_sth_host() {
 }
 
 do_merge() {
-    n=15
+    n=3000
     expected_treesize=$1
     while [ $n -gt 0 ]; do
-        ${top_srcdir}/tools/merge --config machine/merge/catlfish-test.cfg --timing --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
-        ${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+        ${top_srcdir}/tools/merge --config machine/merge-1/catlfish-test.cfg --timing --localconfig current-merge-localconfig.cfg || fail "Merge failed"
+        ${top_srcdir}/tools/mergestatus.py --config machine/merge-1/catlfish-test.cfg --localconfig current-merge-localconfig.cfg
         n=$((n-1))
         treesize=$(get_treesize)
         [ "${treesize}" = "${expected_treesize}" ] && return
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 6ecfcad..d5a22df 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -434,7 +434,27 @@ def gen_config(nodename, config, localconfig):
     plopconfigfile.close()
 
 
-def gen_testmakefile(config, testmakefile, machines, shellvars=False):
+def print_nodevar(configfile, delimiter, nodetype, bootname, names, testaddresses, gennodes=True, gentesturl=True):
+    if gennodes:
+        print >>configfile, nodetype + "NODES=" + delimiter + " ".join(['%s:%s' % (name, bootname) for name in names]) + delimiter
+        for name in names:
+            print >>configfile,  name.upper().replace("-", "_") + "ONLY" + "NODES=" + delimiter + '%s:%s' % (name, bootname) + delimiter
+            if gentesturl:
+                print >>configfile, name.upper().replace("-", "_") + "ONLY" + "TESTURLS=" + delimiter + " ".join(testaddresses[name]) + delimiter
+    else:
+        print >>configfile, nodetype + "NODES=" + delimiter + delimiter
+    if gentesturl:
+        print >>configfile, nodetype + "TESTURLS=" + delimiter + " ".join([address for name in names for address in testaddresses[name]]) + delimiter
+    else:
+        print >>configfile, nodetype + "TESTURLS=" + delimiter + delimiter
+
+def multivaldict(dictlist):
+    d = {}
+    for (key, value) in dictlist:
+        d.setdefault(key, []).append(value)
+    return d
+        
+def gen_testmakefile(config, testmakefile, shellvars=False):
     configfile = open(testmakefile, "w")
     print >>configfile, "#", testmakefile, "generated by", sys.argv[0], datetime.now()
 
@@ -444,27 +464,26 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False):
     mergenodenames = set([node["name"] for node in config["mergenodes"]])
     mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]])
 
-    frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]]
-    storagenodeaddresses = [node["address"] for node in config["storagenodes"]]
-    signingnodeaddresses = [node["address"] for node in config["signingnodes"]]
-    mergesecondarynodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]
+    frontendnodenames_except_last = sorted(frontendnodenames)[:-1]
+    frontendnodenames_except_first = sorted(frontendnodenames)[1:]
+    
+    allnodes = config["frontendnodes"] + config["storagenodes"] + config["signingnodes"] + config["mergenodes"]
 
+    testaddresses = multivaldict([(node["name"], node["address"]) for node in allnodes])
+    
     delimiter = '"' if shellvars else ''
+
+    print_nodevar(configfile, delimiter, "FRONTEND", "catlfish", frontendnodenames, testaddresses)
+    print_nodevar(configfile, delimiter, "FRONTENDEXCEPTLAST", "catlfish", frontendnodenames_except_last, testaddresses)
+    print_nodevar(configfile, delimiter, "FRONTENDEXCEPTFIRST", "catlfish", frontendnodenames_except_first, testaddresses)
+    print_nodevar(configfile, delimiter, "STORAGE", "catlfish", storagenodenames, testaddresses, gennodes=False)
+    print_nodevar(configfile, delimiter, "SIGNING", "catlfish", signingnodenames, testaddresses)
+    print_nodevar(configfile, delimiter, "MERGESECONDARY", "catlfish", mergesecondarynodenames, testaddresses)
+    print_nodevar(configfile, delimiter, "MERGEPRIMARY", "merge", [config["primarymergenode"]], testaddresses, gentesturl=False)
     
-    print >>configfile, "NODES=" + delimiter + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + delimiter
-
-    print >>configfile, "FRONTENDNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in frontendnodenames]) + delimiter
-    print >>configfile, "STORAGENODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in storagenodenames if name not in frontendnodenames]) + delimiter
-    print >>configfile, "SIGNINGNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in signingnodenames]) + delimiter
-    print >>configfile, "MERGESECONDARYNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in mergesecondarynodenames]) + delimiter
-    print >>configfile, "MERGEPRIMARYNODES=" + delimiter + '%s:%s' % (config["primarymergenode"], 'merge') + delimiter
-
-    print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter
-    print >>configfile, "FRONTENDTESTURLS=" + delimiter + " ".join(frontendnodeaddresses) + delimiter
-    print >>configfile, "STORAGETESTURLS=" + delimiter + " ".join(storagenodeaddresses) + delimiter
-    print >>configfile, "SIGNINGTESTURLS=" + delimiter + " ".join(signingnodeaddresses) + delimiter
-    print >>configfile, "MERGESECONDARYTESTURLS=" + delimiter + " ".join(mergesecondarynodeaddresses) + delimiter
-    print >>configfile, "MERGEPRIMARYTESTURLS=" + delimiter + " ".join([]) + delimiter
+    print >>configfile, "NODES=" + delimiter + " ".join(set([node["name"] for node in allnodes])) + delimiter
+
+    print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in frontendnodenames]) + delimiter
     print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter
 
     configfile.close()
@@ -484,15 +503,14 @@ def main():
     parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test")
     parser.add_argument("--testshellvars", metavar="file", help="Generate shell variable file for test")
     parser.add_argument("--getnodenames", action='store_true', help="Get list of node names")
-    parser.add_argument("--machines", type=int, metavar="n", help="Number of machines")
     args = parser.parse_args()
 
-    if args.testmakefile and args.machines:
+    if args.testmakefile:
         config = readconfig.read_config(args.config)
-        gen_testmakefile(config, args.testmakefile, args.machines)
-    elif args.testshellvars and args.machines:
+        gen_testmakefile(config, args.testmakefile)
+    elif args.testshellvars:
         config = readconfig.read_config(args.config)
-        gen_testmakefile(config, args.testshellvars, args.machines, shellvars=True)
+        gen_testmakefile(config, args.testshellvars, shellvars=True)
     elif args.getnodenames:
         config = readconfig.read_config(args.config)
         printnodenames(config)
diff --git a/tools/testcase1.py b/tools/testcase1.py
index fb081e5..a616841 100755
--- a/tools/testcase1.py
+++ b/tools/testcase1.py
@@ -192,11 +192,11 @@ def mergestatus(configfile, localconfigfile):
 
 def merge(expected=None, wait=0):
     rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg",
-                          "--localconfig", testdir + "/catlfish-test-local-merge.cfg"])
+                          "--localconfig", testdir + "/catlfish-test-local-merge-1.cfg"])
     if rv:
         return rv
     for i in range(10):
-        mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge.cfg")
+        mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge-1.cfg")
         if i < wait:
             sleep(1)
             continue
@@ -214,6 +214,7 @@ for baseurl in baseurls:
 testgroup("cert1")
 
 result1 = do_add_chain(cc1, baseurls[0])
+result2 = do_add_chain(cc1, baseurls[0])
 
 mergeresult = merge(expected=1)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
@@ -224,9 +225,10 @@ for baseurl in baseurls:
     print_and_check_tree_size(1, baseurl)
 size_sth[1] = base64.b64decode(get_sth(baseurls[0])["sha256_root_hash"])
 
-result2 = do_add_chain(cc1, baseurls[0])
+result3 = do_add_chain(cc1, baseurls[0])
 
 assert_equal(result2["timestamp"], result1["timestamp"], "timestamp")
+assert_equal(result3["timestamp"], result1["timestamp"], "timestamp")
 
 mergeresult = merge(expected=1, wait=3)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)