Added tools for getting and generating global config

12 files changed, 73 insertions, 29 deletions

diff --git a/test/catlfish-test-local-1.cfg b/test/catlfish-test-local-1.cfg
index 1795649..21f8c17 100644
--- a/test/catlfish-test-local-1.cfg
+++ b/test/catlfish-test-local-1.cfg
@@ -1,6 +1,5 @@
 localnodes:
         - frontend-1
-        - storage-1
 
 frontendaddresses:
         frontend-1: 127.0.0.1:8082
@@ -9,7 +8,7 @@ ctapiaddresses:
         frontend-1: 127.0.0.1:8080
 
 storageaddresses:
-        storage-1: 127.0.0.1:8081
+        frontend-1: 127.0.0.1:8081
 
 
 #publichttpaddresses:
@@ -34,5 +33,7 @@ logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
     Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
     0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
 
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
+
 #options:
 #        - sctcaching
diff --git a/test/catlfish-test-local-merge-2.cfg b/test/catlfish-test-local-merge-2.cfg
index b871313..e2baa01 100644
--- a/test/catlfish-test-local-merge-2.cfg
+++ b/test/catlfish-test-local-merge-2.cfg
@@ -22,3 +22,5 @@ paths:
 logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
     Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
     0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg
index e0841e9..9283136 100644
--- a/test/catlfish-test-local-merge.cfg
+++ b/test/catlfish-test-local-merge.cfg
@@ -28,3 +28,5 @@ merge:
         dist-window-size: 2
         dist-sendlog-chunksize: 1
         dist-sendentries-chunksize: 1
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-signing.cfg b/test/catlfish-test-local-signing.cfg
index df91bcd..0eccc94 100644
--- a/test/catlfish-test-local-signing.cfg
+++ b/test/catlfish-test-local-signing.cfg
@@ -23,3 +23,5 @@ paths:
 logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
     Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
     0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test.cfg.in b/test/catlfish-test.cfg.in
index ed3494b..68c753f 100644
--- a/test/catlfish-test.cfg.in
+++ b/test/catlfish-test.cfg.in
@@ -6,7 +6,7 @@ frontendnodes:
           address: localhost:8082
 
 storagenodes:
-        - name: storage-1
+        - name: frontend-1
           address: localhost:8081
 
 signingnodes:
diff --git a/test/scripts/light-system-test-prepare-redistribute-frontend.sh b/test/scripts/light-system-test-prepare-redistribute-frontend.sh
index cba93d4..7141ba5 100755
--- a/test/scripts/light-system-test-prepare-redistribute-frontend.sh
+++ b/test/scripts/light-system-test-prepare-redistribute-frontend.sh
@@ -11,3 +11,4 @@ mv machine/machine-1 machine/machine-1-down
 mkdir -p machine/machine-1/db
 touch machine/machine-1/db/index
 touch machine/machine-1/db/newentries
+cp machine/machine-1-down/{catlfish-test-local-1.cfg,catlfish-test.cfg,catlfish-test.cfg.sig} machine/machine-1/
diff --git a/test/scripts/light-system-test-prepare.sh b/test/scripts/light-system-test-prepare.sh
index 536c0db..6c706fd 100755
--- a/test/scripts/light-system-test-prepare.sh
+++ b/test/scripts/light-system-test-prepare.sh
@@ -38,7 +38,6 @@ createcert () {
 
 createca
 createcert
-cafingerprint=$(openssl x509 -in httpsca/demoCA/cacert.pem -noout -sha256 -fingerprint | sed -e 's/.*=//' -e 's/://g')
 mkdir keys
 (cd keys ; ${top_srcdir}/tools/create-key.sh logkey)
 openssl pkcs8 -topk8 -nocrypt -in keys/logkey-private.pem -out keys/logkey-private.pkcs8
@@ -50,29 +49,32 @@ printf 0 > mergedb-secondary/verifiedsize
 mkdir known_roots
 cp ${top_srcdir}/tools/testcerts/roots/* known_roots
 mkdir privatekeys
-echo "apikeys:" > api-keys.cfg
+
 for node in ${NODES}; do \
     (cd privatekeys ; ${top_srcdir}/tools/create-key.sh ${node})
-    apipk=$(grep -v '^-----' privatekeys/${node}.pem | tr '\n' ' ')
     mkdir -p nodes/${node}/log
-    echo "        - nodename: ${node}" >> api-keys.cfg
-    echo "          publickey: ${apipk}" >> api-keys.cfg
 done
 
-logpk=$(grep -v '^-----' keys/logkey.pem | tr '\n' ' ')
-echo "logpublickey: ${logpk}" >> api-keys.cfg
-echo "cafingerprint: ${cafingerprint}" >> api-keys.cfg
-
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
 
-cat ${top_srcdir}/test/catlfish-test.cfg.in api-keys.cfg > ${top_srcdir}/test/catlfish-test.cfg
-openssl dgst -sha256 -sign ${top_srcdir}/test/logadminkey-private.pem -out ${top_srcdir}/test/catlfish-test.cfg.sig ${top_srcdir}/test/catlfish-test.cfg
 for machine in ${MACHINES}; do \
-    ${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-${machine}.cfg
     mkdir -p machine/machine-${machine}/db
+    cp ${top_srcdir}/test/catlfish-test-local-${machine}.cfg machine/machine-${machine}/
+    ${top_srcdir}/tools/getconfig.py --localconfig machine/machine-${machine}/catlfish-test-local-${machine}.cfg --dest machine/machine-${machine}/catlfish-test.cfg
+    ${top_srcdir}/tools/compileconfig.py --config machine/machine-${machine}/catlfish-test.cfg --localconfig machine/machine-${machine}/catlfish-test-local-${machine}.cfg
     touch machine/machine-${machine}/db/index && touch machine/machine-${machine}/db/newentries
 done
-${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg
-${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge-2.cfg
-${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-signing.cfg
+mkdir -p machine/merge
+cp ${top_srcdir}/test/catlfish-test-local-merge.cfg machine/merge/
+${top_srcdir}/tools/getconfig.py --localconfig machine/merge/catlfish-test-local-merge.cfg --dest machine/merge/catlfish-test.cfg
+${top_srcdir}/tools/compileconfig.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+mkdir -p machine/merge-2
+cp ${top_srcdir}/test/catlfish-test-local-merge-2.cfg machine/merge-2/
+${top_srcdir}/tools/getconfig.py --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg --dest machine/merge-2/catlfish-test.cfg
+${top_srcdir}/tools/compileconfig.py --config machine/merge-2/catlfish-test.cfg --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg
+mkdir -p machine/signing
+cp ${top_srcdir}/test/catlfish-test-local-signing.cfg machine/signing/
+${top_srcdir}/tools/getconfig.py --localconfig machine/signing/catlfish-test-local-signing.cfg --dest machine/signing/catlfish-test.cfg
+${top_srcdir}/tools/compileconfig.py --config machine/signing/catlfish-test.cfg --localconfig machine/signing/catlfish-test-local-signing.cfg
 test -x ${SOFTHSM} && ${SOFTHSM} --init-token --slot=0 --label=mylabel --so-pin=ffff --pin=ffff || true
 test -x ${SOFTHSM} && ${SOFTHSM} --import keys/logkey-private.pkcs8 --slot 0 --label mylabel --pin ffff --id 00 || true
diff --git a/test/scripts/light-system-test-run-1.sh b/test/scripts/light-system-test-run-1.sh
index 3b6266e..1ad56a8 100755
--- a/test/scripts/light-system-test-run-1.sh
+++ b/test/scripts/light-system-test-run-1.sh
@@ -7,14 +7,14 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 . ${top_srcdir}/test/scripts/testutils.sh
 
-python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test || fail "Tests failed"
+python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem machine/merge || fail "Tests failed"
 check_sth
 python ${top_srcdir}/tools/fetchallcerts.py ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification failed"
-python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg  || fail "GC failed"
 for certfile in ${top_srcdir}/tools/testcerts/cert[1-5].txt ${top_srcdir}/tools/testcerts/pre[12].txt; do
     python ${top_srcdir}/tools/submitcert.py --parallel=1 --store $certfile --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 done
-python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg  || fail "GC failed"
 do_merge
 check_sth
 
@@ -30,16 +30,16 @@ python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert3.tx
 python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert4.txt fetchcertstore/00000003 || fail "Verification failed"
 python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert5.txt fetchcertstore/00000004 || fail "Verification failed"
 python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/pre1.txt:${top_srcdir}/tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006 || fail"Verification failed"
-python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed"
 
 ${top_srcdir}/tools/to_catlfish.py to_erl nodes/merge-2/ "init:stop()"
 python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert6.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 echo NOTE: merge backup should fail with 111 Connection refused
 
-${top_srcdir}/tools/merge_fetch.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed"
-${top_srcdir}/tools/merge_backup.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg && fail "Merge backup should have failed"
-${top_srcdir}/tools/merge_sth.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed"
-${top_srcdir}/tools/merge_dist.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed"
+${top_srcdir}/tools/merge_fetch.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
+${top_srcdir}/tools/merge_backup.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg && fail "Merge backup should have failed"
+${top_srcdir}/tools/merge_sth.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
+${top_srcdir}/tools/merge_dist.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
 
 assert_equal "Tree size" "$(get_treesize)" 7
 
diff --git a/test/scripts/light-system-test-run-2.sh b/test/scripts/light-system-test-run-2.sh
index 4c8bbdb..bf80c97 100755
--- a/test/scripts/light-system-test-run-2.sh
+++ b/test/scripts/light-system-test-run-2.sh
@@ -11,4 +11,4 @@ python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts --parallel 1 $
 
 check_sth
 
-python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg  || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg  || fail "GC failed"
diff --git a/test/scripts/light-system-test-run-5.sh b/test/scripts/light-system-test-run-5.sh
new file mode 100755
index 0000000..355e179
--- /dev/null
+++ b/test/scripts/light-system-test-run-5.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -o nounset
+set -o errexit
+
+top_srcdir=$(cd $(dirname $0)/../..; pwd)
+
+. ${top_srcdir}/test/scripts/testutils.sh
+
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+
+for machine in ${MACHINES}; do \
+    ${top_srcdir}/tools/getconfig.sh machine/machine-${machine}/catlfish-test-local-${machine}.cfg machine/machine-${machine}/catlfish-test.cfg frontend-${machine}.plopcontrol
+done
+
+
+do_merge
+check_sth
+
+assert_equal "Tree size" "$(get_treesize)" 9
diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh
index 816bd67..1f761e8 100755
--- a/test/scripts/light-system-test.sh
+++ b/test/scripts/light-system-test.sh
@@ -68,5 +68,19 @@ echo
 ${SCRIPTS}/light-system-test-prepare-redistribute-frontend.sh
 tests_start
 ${SCRIPTS}/light-system-test-run-4.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
+
+echo
+echo
+echo
+echo
+echo Test 5
+echo
+echo
+echo
+echo
+echo
+
+
+${SCRIPTS}/light-system-test-run-5.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
 sleep 5
 tests_stop
diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh
index ba4221c..731b6e5 100644
--- a/test/scripts/testutils.sh
+++ b/test/scripts/testutils.sh
@@ -25,7 +25,7 @@ get_treesize() {
     get_treesize_host ${BASEURL}
 }
 get_treesize_host() {
-    ${top_srcdir}/tools/loginfo.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg --treesize "$1"
+    ${top_srcdir}/tools/loginfo.py --localconfig machine/merge/catlfish-test-local-merge.cfg --treesize "$1"
 }
 
 check_sth() {
@@ -36,7 +36,7 @@ check_sth_host() {
 }
 
 do_merge() {
-    ${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --timing --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed"
+    ${top_srcdir}/tools/merge --config machine/merge/catlfish-test.cfg --timing --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
     sleep 5                     # FIXME: Just wait for dist instead.
 }