From 8a1f3a5f5b1458aceb7567add95e7637e9c2e5fc Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 2 Feb 2017 16:08:24 +0100 Subject: Added tools for getting and generating global config --- test/catlfish-test-local-1.cfg | 5 +- test/catlfish-test-local-merge-2.cfg | 2 + test/catlfish-test-local-merge.cfg | 2 + test/catlfish-test-local-signing.cfg | 2 + test/catlfish-test.cfg.in | 2 +- ...ht-system-test-prepare-redistribute-frontend.sh | 1 + test/scripts/light-system-test-prepare.sh | 32 +++++----- test/scripts/light-system-test-run-1.sh | 16 ++--- test/scripts/light-system-test-run-2.sh | 2 +- test/scripts/light-system-test-run-5.sh | 20 +++++++ test/scripts/light-system-test.sh | 14 +++++ test/scripts/testutils.sh | 4 +- tools/compileconfig.py | 13 +++++ tools/genconfig.sh | 43 ++++++++++++++ tools/getconfig.py | 68 ++++++++++++++++++++++ tools/getconfig.sh | 29 +++++++++ tools/loginfo.py | 1 - tools/readconfig.py | 13 +++-- 18 files changed, 235 insertions(+), 34 deletions(-) create mode 100755 test/scripts/light-system-test-run-5.sh create mode 100755 tools/genconfig.sh create mode 100755 tools/getconfig.py create mode 100755 tools/getconfig.sh diff --git a/test/catlfish-test-local-1.cfg b/test/catlfish-test-local-1.cfg index 1795649..21f8c17 100644 --- a/test/catlfish-test-local-1.cfg +++ b/test/catlfish-test-local-1.cfg @@ -1,6 +1,5 @@ localnodes: - frontend-1 - - storage-1 frontendaddresses: frontend-1: 127.0.0.1:8082 @@ -9,7 +8,7 @@ ctapiaddresses: frontend-1: 127.0.0.1:8080 storageaddresses: - storage-1: 127.0.0.1:8081 + frontend-1: 127.0.0.1:8081 #publichttpaddresses: @@ -34,5 +33,7 @@ logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+ 0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ== +configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg + #options: # - sctcaching diff --git a/test/catlfish-test-local-merge-2.cfg b/test/catlfish-test-local-merge-2.cfg index b871313..e2baa01 100644 --- a/test/catlfish-test-local-merge-2.cfg +++ b/test/catlfish-test-local-merge-2.cfg @@ -22,3 +22,5 @@ paths: logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+ 0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ== + +configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg index e0841e9..9283136 100644 --- a/test/catlfish-test-local-merge.cfg +++ b/test/catlfish-test-local-merge.cfg @@ -28,3 +28,5 @@ merge: dist-window-size: 2 dist-sendlog-chunksize: 1 dist-sendentries-chunksize: 1 + +configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg diff --git a/test/catlfish-test-local-signing.cfg b/test/catlfish-test-local-signing.cfg index df91bcd..0eccc94 100644 --- a/test/catlfish-test-local-signing.cfg +++ b/test/catlfish-test-local-signing.cfg @@ -23,3 +23,5 @@ paths: logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+ 0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ== + +configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg diff --git a/test/catlfish-test.cfg.in b/test/catlfish-test.cfg.in index ed3494b..68c753f 100644 --- a/test/catlfish-test.cfg.in +++ b/test/catlfish-test.cfg.in @@ -6,7 +6,7 @@ frontendnodes: address: localhost:8082 storagenodes: - - name: storage-1 + - name: frontend-1 address: localhost:8081 signingnodes: diff --git a/test/scripts/light-system-test-prepare-redistribute-frontend.sh b/test/scripts/light-system-test-prepare-redistribute-frontend.sh index cba93d4..7141ba5 100755 --- a/test/scripts/light-system-test-prepare-redistribute-frontend.sh +++ b/test/scripts/light-system-test-prepare-redistribute-frontend.sh @@ -11,3 +11,4 @@ mv machine/machine-1 machine/machine-1-down mkdir -p machine/machine-1/db touch machine/machine-1/db/index touch machine/machine-1/db/newentries +cp machine/machine-1-down/{catlfish-test-local-1.cfg,catlfish-test.cfg,catlfish-test.cfg.sig} machine/machine-1/ diff --git a/test/scripts/light-system-test-prepare.sh b/test/scripts/light-system-test-prepare.sh index 536c0db..6c706fd 100755 --- a/test/scripts/light-system-test-prepare.sh +++ b/test/scripts/light-system-test-prepare.sh @@ -38,7 +38,6 @@ createcert () { createca createcert -cafingerprint=$(openssl x509 -in httpsca/demoCA/cacert.pem -noout -sha256 -fingerprint | sed -e 's/.*=//' -e 's/://g') mkdir keys (cd keys ; ${top_srcdir}/tools/create-key.sh logkey) openssl pkcs8 -topk8 -nocrypt -in keys/logkey-private.pem -out keys/logkey-private.pkcs8 @@ -50,29 +49,32 @@ printf 0 > mergedb-secondary/verifiedsize mkdir known_roots cp ${top_srcdir}/tools/testcerts/roots/* known_roots mkdir privatekeys -echo "apikeys:" > api-keys.cfg + for node in ${NODES}; do \ (cd privatekeys ; ${top_srcdir}/tools/create-key.sh ${node}) - apipk=$(grep -v '^-----' privatekeys/${node}.pem | tr '\n' ' ') mkdir -p nodes/${node}/log - echo " - nodename: ${node}" >> api-keys.cfg - echo " publickey: ${apipk}" >> api-keys.cfg done -logpk=$(grep -v '^-----' keys/logkey.pem | tr '\n' ' ') -echo "logpublickey: ${logpk}" >> api-keys.cfg -echo "cafingerprint: ${cafingerprint}" >> api-keys.cfg - +${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg -cat ${top_srcdir}/test/catlfish-test.cfg.in api-keys.cfg > ${top_srcdir}/test/catlfish-test.cfg -openssl dgst -sha256 -sign ${top_srcdir}/test/logadminkey-private.pem -out ${top_srcdir}/test/catlfish-test.cfg.sig ${top_srcdir}/test/catlfish-test.cfg for machine in ${MACHINES}; do \ - ${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-${machine}.cfg mkdir -p machine/machine-${machine}/db + cp ${top_srcdir}/test/catlfish-test-local-${machine}.cfg machine/machine-${machine}/ + ${top_srcdir}/tools/getconfig.py --localconfig machine/machine-${machine}/catlfish-test-local-${machine}.cfg --dest machine/machine-${machine}/catlfish-test.cfg + ${top_srcdir}/tools/compileconfig.py --config machine/machine-${machine}/catlfish-test.cfg --localconfig machine/machine-${machine}/catlfish-test-local-${machine}.cfg touch machine/machine-${machine}/db/index && touch machine/machine-${machine}/db/newentries done -${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg -${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge-2.cfg -${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-signing.cfg +mkdir -p machine/merge +cp ${top_srcdir}/test/catlfish-test-local-merge.cfg machine/merge/ +${top_srcdir}/tools/getconfig.py --localconfig machine/merge/catlfish-test-local-merge.cfg --dest machine/merge/catlfish-test.cfg +${top_srcdir}/tools/compileconfig.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg +mkdir -p machine/merge-2 +cp ${top_srcdir}/test/catlfish-test-local-merge-2.cfg machine/merge-2/ +${top_srcdir}/tools/getconfig.py --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg --dest machine/merge-2/catlfish-test.cfg +${top_srcdir}/tools/compileconfig.py --config machine/merge-2/catlfish-test.cfg --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg +mkdir -p machine/signing +cp ${top_srcdir}/test/catlfish-test-local-signing.cfg machine/signing/ +${top_srcdir}/tools/getconfig.py --localconfig machine/signing/catlfish-test-local-signing.cfg --dest machine/signing/catlfish-test.cfg +${top_srcdir}/tools/compileconfig.py --config machine/signing/catlfish-test.cfg --localconfig machine/signing/catlfish-test-local-signing.cfg test -x ${SOFTHSM} && ${SOFTHSM} --init-token --slot=0 --label=mylabel --so-pin=ffff --pin=ffff || true test -x ${SOFTHSM} && ${SOFTHSM} --import keys/logkey-private.pkcs8 --slot 0 --label mylabel --pin ffff --id 00 || true diff --git a/test/scripts/light-system-test-run-1.sh b/test/scripts/light-system-test-run-1.sh index 3b6266e..1ad56a8 100755 --- a/test/scripts/light-system-test-run-1.sh +++ b/test/scripts/light-system-test-run-1.sh @@ -7,14 +7,14 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd) . ${top_srcdir}/test/scripts/testutils.sh -python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test || fail "Tests failed" +python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem machine/merge || fail "Tests failed" check_sth python ${top_srcdir}/tools/fetchallcerts.py ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification failed" -python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg || fail "GC failed" +python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed" for certfile in ${top_srcdir}/tools/testcerts/cert[1-5].txt ${top_srcdir}/tools/testcerts/pre[12].txt; do python ${top_srcdir}/tools/submitcert.py --parallel=1 --store $certfile --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed" done -python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg || fail "GC failed" +python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed" do_merge check_sth @@ -30,16 +30,16 @@ python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert3.tx python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert4.txt fetchcertstore/00000003 || fail "Verification failed" python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert5.txt fetchcertstore/00000004 || fail "Verification failed" python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/pre1.txt:${top_srcdir}/tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006 || fail"Verification failed" -python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg || fail "GC failed" +python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed" ${top_srcdir}/tools/to_catlfish.py to_erl nodes/merge-2/ "init:stop()" python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert6.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed" echo NOTE: merge backup should fail with 111 Connection refused -${top_srcdir}/tools/merge_fetch.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed" -${top_srcdir}/tools/merge_backup.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg && fail "Merge backup should have failed" -${top_srcdir}/tools/merge_sth.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed" -${top_srcdir}/tools/merge_dist.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed" +${top_srcdir}/tools/merge_fetch.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed" +${top_srcdir}/tools/merge_backup.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg && fail "Merge backup should have failed" +${top_srcdir}/tools/merge_sth.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed" +${top_srcdir}/tools/merge_dist.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed" assert_equal "Tree size" "$(get_treesize)" 7 diff --git a/test/scripts/light-system-test-run-2.sh b/test/scripts/light-system-test-run-2.sh index 4c8bbdb..bf80c97 100755 --- a/test/scripts/light-system-test-run-2.sh +++ b/test/scripts/light-system-test-run-2.sh @@ -11,4 +11,4 @@ python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts --parallel 1 $ check_sth -python ${top_srcdir}/tools/storagegc.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-1.cfg || fail "GC failed" +python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed" diff --git a/test/scripts/light-system-test-run-5.sh b/test/scripts/light-system-test-run-5.sh new file mode 100755 index 0000000..355e179 --- /dev/null +++ b/test/scripts/light-system-test-run-5.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -o nounset +set -o errexit + +top_srcdir=$(cd $(dirname $0)/../..; pwd) + +. ${top_srcdir}/test/scripts/testutils.sh + +${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg + +for machine in ${MACHINES}; do \ + ${top_srcdir}/tools/getconfig.sh machine/machine-${machine}/catlfish-test-local-${machine}.cfg machine/machine-${machine}/catlfish-test.cfg frontend-${machine}.plopcontrol +done + + +do_merge +check_sth + +assert_equal "Tree size" "$(get_treesize)" 9 diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh index 816bd67..1f761e8 100755 --- a/test/scripts/light-system-test.sh +++ b/test/scripts/light-system-test.sh @@ -68,5 +68,19 @@ echo ${SCRIPTS}/light-system-test-prepare-redistribute-frontend.sh tests_start ${SCRIPTS}/light-system-test-run-4.sh || (echo "Tests failed"; sleep 5; tests_stop; false) + +echo +echo +echo +echo +echo Test 5 +echo +echo +echo +echo +echo + + +${SCRIPTS}/light-system-test-run-5.sh || (echo "Tests failed"; sleep 5; tests_stop; false) sleep 5 tests_stop diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh index ba4221c..731b6e5 100644 --- a/test/scripts/testutils.sh +++ b/test/scripts/testutils.sh @@ -25,7 +25,7 @@ get_treesize() { get_treesize_host ${BASEURL} } get_treesize_host() { - ${top_srcdir}/tools/loginfo.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg --treesize "$1" + ${top_srcdir}/tools/loginfo.py --localconfig machine/merge/catlfish-test-local-merge.cfg --treesize "$1" } check_sth() { @@ -36,7 +36,7 @@ check_sth_host() { } do_merge() { - ${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --timing --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed" + ${top_srcdir}/tools/merge --config machine/merge/catlfish-test.cfg --timing --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed" sleep 5 # FIXME: Just wait for dist instead. } diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 47fd19a..abe59a3 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -404,6 +404,7 @@ def gen_config(nodename, config, localconfig): (Symbol("allowed_clients"), list(allowed_clients)), (Symbol("allowed_servers"), list(allowed_servers)), (Symbol("apikeys"), apikeys), + (Symbol("version"), config["version"]), ] erlangconfig = [ @@ -445,12 +446,21 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False): configfile.close() +def printnodenames(config): + frontendnodenames = set([node["name"] for node in config["frontendnodes"]]) + storagenodenames = set([node["name"] for node in config["storagenodes"]]) + signingnodenames = set([node["name"] for node in config["signingnodes"]]) + mergenodenames = set([node["name"] for node in config["mergenodes"]]) + + print " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + def main(): parser = argparse.ArgumentParser(description="") parser.add_argument('--config', help="System configuration", required=True) parser.add_argument('--localconfig', help="Local configuration") parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test") parser.add_argument("--testshellvars", metavar="file", help="Generate shell variable file for test") + parser.add_argument("--getnodenames", action='store_true', help="Get list of node names") parser.add_argument("--machines", type=int, metavar="n", help="Number of machines") args = parser.parse_args() @@ -460,6 +470,9 @@ def main(): elif args.testshellvars and args.machines: config = readconfig.read_config(args.config) gen_testmakefile(config, args.testshellvars, args.machines, shellvars=True) + elif args.getnodenames: + config = readconfig.read_config(args.config) + printnodenames(config) elif args.localconfig: localconfig = readconfig.read_config(args.localconfig) config = readconfig.verify_and_read_config(args.config, localconfig["logadminkey"]) diff --git a/tools/genconfig.sh b/tools/genconfig.sh new file mode 100755 index 0000000..89bcd85 --- /dev/null +++ b/tools/genconfig.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +# usage: genconfig.sh