diff options
| -rw-r--r-- | test/Makefile | 1 | ||||
| -rw-r--r-- | test/catlfish-test.cfg.in (renamed from test/catlfish-test.cfg) | 0 | ||||
| -rwxr-xr-x | test/scripts/light-system-test-prepare.sh | 29 | ||||
| -rw-r--r-- | tools/certtools.py | 24 | ||||
| -rwxr-xr-x | tools/compileconfig.py | 14 | ||||
| -rw-r--r-- | tools/mergetools.py | 38 |
6 files changed, 65 insertions, 41 deletions
diff --git a/test/Makefile b/test/Makefile index c92c30d..f311208 100644 --- a/test/Makefile +++ b/test/Makefile @@ -11,6 +11,7 @@ tests-wait: sleep 5 tests-makemk: + cat $(PREFIX)/test/catlfish-test.cfg.in | sed 's/@[A-Z0-9-]*@//' > $(PREFIX)/test/catlfish-test.cfg $(PREFIX)/tools/compileconfig.py --config=$(PREFIX)/test/catlfish-test.cfg --testshellvars=$(PREFIX)/test/test.shvars --machines 1 tests: diff --git a/test/catlfish-test.cfg b/test/catlfish-test.cfg.in index 39288c7..39288c7 100644 --- a/test/catlfish-test.cfg +++ b/test/catlfish-test.cfg.in diff --git a/test/scripts/light-system-test-prepare.sh b/test/scripts/light-system-test-prepare.sh index 6f6dd07..df45d25 100755 --- a/test/scripts/light-system-test-prepare.sh +++ b/test/scripts/light-system-test-prepare.sh @@ -38,6 +38,7 @@ createcert () { createca createcert +cafingerprint=$(openssl x509 -in httpsca/demoCA/cacert.pem -noout -sha256 -fingerprint | sed -e 's/.*=//' -e 's/://g') mkdir keys (cd keys ; ${top_srcdir}/tools/create-key.sh logkey) openssl pkcs8 -topk8 -nocrypt -in keys/logkey-private.pem -out keys/logkey-private.pkcs8 @@ -48,6 +49,23 @@ touch mergedb-secondary/logorder printf 0 > mergedb-secondary/verifiedsize mkdir known_roots cp ${top_srcdir}/tools/testcerts/roots/* known_roots +mkdir privatekeys +mkdir publickeys +echo "apikeys:" > api-keys.cfg +for node in ${NODES}; do \ + (cd privatekeys ; ${top_srcdir}/tools/create-key.sh ${node}) + apipk=$(grep -v '^-----' privatekeys/${node}.pem | tr '\n' ' ') + mkdir -p nodes/${node}/log + echo " - nodename: ${node}" >> api-keys.cfg + echo " publickey: ${apipk}" >> api-keys.cfg +done + +logpk=$(grep -v '^-----' keys/logkey.pem | tr '\n' ' ') +echo "logpublickey: ${logpk}" >> api-keys.cfg +echo "cafingerprint: ${cafingerprint}" >> api-keys.cfg + + +cat ${top_srcdir}/test/catlfish-test.cfg.in api-keys.cfg > ${top_srcdir}/test/catlfish-test.cfg for machine in ${MACHINES}; do \ ${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-${machine}.cfg mkdir -p machine/machine-${machine}/db @@ -55,16 +73,5 @@ for machine in ${MACHINES}; do \ done ${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge-2.cfg ${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-signing.cfg -mkdir privatekeys -mkdir publickeys -for node in ${NODES}; do \ - (cd privatekeys ; ${top_srcdir}/tools/create-key.sh ${node}) ; \ - mv privatekeys/${node}.pem publickeys/ ; \ - mkdir -p nodes/${node}/log -done -(cd privatekeys ; ${top_srcdir}/tools/create-key.sh merge-1) -mv privatekeys/merge-1.pem publickeys/ -(cd privatekeys ; ${top_srcdir}/tools/create-key.sh merge-2) -mv privatekeys/merge-2.pem publickeys/ test -x ${SOFTHSM} && ${SOFTHSM} --init-token --slot=0 --label=mylabel --so-pin=ffff --pin=ffff || true test -x ${SOFTHSM} && ${SOFTHSM} --import keys/logkey-private.pkcs8 --slot 0 --label mylabel --pin ffff --id 00 || true diff --git a/tools/certtools.py b/tools/certtools.py index 0009d5d..0ccbcad 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -94,6 +94,12 @@ def get_root_cert(issuer): class sslparameters: cafile = None +class apikeys: + publickeys = {} + +def set_api_keys(config): + apikeys.publickeys = dict([(node["nodename"], base64.b64decode(node["publickey"])) for node in config["apikeys"]]) + def create_ssl_context(cafile=None): try: sslparameters.cafile = cafile @@ -256,32 +262,32 @@ def parse_auth_header(authheader): options = dict([(e.partition("=")[0], e.partition("=")[2]) for e in rawoptions]) return (base64.b64decode(signature), options) -def check_auth_header(authheader, expected_key, publickeydir, data, path): +def check_auth_header(authheader, expected_key, data, path): if expected_key == None: return True (signature, options) = parse_auth_header(authheader) + publickey = apikeys.publickeys[expected_key] keyname = options.get("key") if keyname != expected_key: raise Exception("Response claimed to come from %s, expected %s" % (keyname, expected_key)) - publickey = get_public_key_from_file(publickeydir + "/" + keyname + ".pem") vk = ecdsa.VerifyingKey.from_der(publickey) vk.verify(signature, "%s\0%s\0%s" % ("REPLY", path, data), hashfunc=hashlib.sha256, sigdecode=ecdsa.util.sigdecode_der) return True -def http_request(url, data=None, key=None, verifynode=None, publickeydir=".", params=None, session=None): +def http_request(url, data=None, key=None, verifynode=None, params=None, session=None): if session: - return http_request_session(url, data=data, key=key, verifynode=verifynode, publickeydir=publickeydir, params=params, session=session) + return http_request_session(url, data=data, key=key, verifynode=verifynode, params=params, session=session) else: with requests.sessions.Session() as session: - return http_request_session(url, data=data, key=key, verifynode=verifynode, publickeydir=publickeydir, params=params, session=session) + return http_request_session(url, data=data, key=key, verifynode=verifynode, params=params, session=session) def chunk_generator(data, maxsize): while len(data): yield data[:maxsize] data = data[maxsize:] -def http_request_session(url, data=None, key=None, verifynode=None, publickeydir=".", params=None, session=None): +def http_request_session(url, data=None, key=None, verifynode=None, params=None, session=None): (keyname, keyfile) = key privatekey = get_eckey_from_file(keyfile) sk = ecdsa.SigningKey.from_der(privatekey) @@ -312,7 +318,7 @@ def http_request_session(url, data=None, key=None, verifynode=None, publickeydir result.raise_for_status() authheader = result.headers.get('X-Catlfish-Auth') data = result.text - check_auth_header(authheader, verifynode, publickeydir, data, url_to_sign) + check_auth_header(authheader, verifynode, data, url_to_sign) return data def get_signature(baseurl, data, key=None): @@ -431,8 +437,10 @@ def timing_point(timer_dict=None, name=None): starttime = timer_dict["lasttime"] stoptime = t deltatime = stoptime - starttime - timer_dict["deltatimes"].append((name, deltatime.seconds * 1000000 + deltatime.microseconds)) + microseconds = deltatime.seconds * 1000000 + deltatime.microseconds + timer_dict["deltatimes"].append((name, microseconds)) timer_dict["lasttime"] = t + #print name, microseconds/1000000.0 return None else: timer_dict = {"deltatimes":[], "lasttime":t} diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 1fa352e..bbc2277 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -7,10 +7,14 @@ import argparse import sys import yaml import re +import base64 class Symbol(str): pass +class Binary(str): + pass + clean_string = re.compile(r'^[-.:_/A-Za-z0-9 ]*$') clean_symbol = re.compile(r'^[_A-Za-z0-9]*$') @@ -34,6 +38,8 @@ def gen_erlang(term, level=1): separator = ",\n" + indent if isinstance(term, Symbol): return quote_erlang_symbol(term) + elif isinstance(term, Binary): + return "<<" + ",".join([str(ord(c)) for c in term]) + ">>" elif isinstance(term, basestring): return quote_erlang_string(term) elif isinstance(term, int): @@ -194,9 +200,13 @@ def parse_ratelimit((type, description)): print >>sys.stderr, "%s: Only one ratelimit expression supported right now" % (type,) return (Symbol(type), descriptions) +def api_keys(config): + return [(node["nodename"], Binary(base64.b64decode(node["publickey"]))) for node in config["apikeys"]] + def gen_config(nodename, config, localconfig): print "generating config for", nodename paths = localconfig["paths"] + apikeys = api_keys(config) bind_addresses = { "frontend": localconfig.get("frontendaddresses", {}).get(nodename), "storage": localconfig.get("storageaddresses", {}).get(nodename), @@ -231,6 +241,7 @@ def gen_config(nodename, config, localconfig): (Symbol("https_certfile"), paths["https_certfile"]), (Symbol("https_keyfile"), paths["https_keyfile"]), (Symbol("https_cacertfile"), paths["https_cacertfile"]), + (Symbol("https_cacert_fingerprint"), Binary(base64.b16decode(config["cafingerprint"]))), ] catlfishconfig.append((Symbol("mmd"), config["mmd"])) @@ -333,7 +344,7 @@ def gen_config(nodename, config, localconfig): print >>sys.stderr, "Neither logprivatekey nor hsm configured for signing node", nodename sys.exit(1) plopconfig += [ - (Symbol("log_public_key"), paths["logpublickey"]), + (Symbol("log_public_key"), Binary(base64.b64decode(config["logpublickey"]))), (Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))), ] if "frontendnodes" in nodetype: @@ -341,6 +352,7 @@ def gen_config(nodename, config, localconfig): plopconfig += [ (Symbol("allowed_clients"), list(allowed_clients)), (Symbol("allowed_servers"), list(allowed_servers)), + (Symbol("apikeys"), apikeys), ] erlangconfig = [ diff --git a/tools/mergetools.py b/tools/mergetools.py index 94901a9..19d16ca 100644 --- a/tools/mergetools.py +++ b/tools/mergetools.py @@ -14,7 +14,7 @@ try: import permdb except ImportError: pass -from certtools import get_leaf_hash, http_request, get_leaf_hash +from certtools import get_leaf_hash, http_request, get_leaf_hash, set_api_keys def parselogrow(row): return base64.b16decode(row, casefold=True) @@ -167,8 +167,7 @@ def fsync_logorder(logorderfile): def get_new_entries(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/storage/fetchnewentries", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return [base64.b64decode(entry) for \ @@ -185,7 +184,7 @@ def get_entries(node, baseurl, own_key, paths, hashes, session=None): result = http_request(baseurl + "plop/v1/storage/getentry", params=params, key=own_key, verifynode=node, - publickeydir=paths["publickeys"], session=session) + session=session) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": entries = dict([(base64.b64decode(entry["hash"]), @@ -203,8 +202,7 @@ def get_entries(node, baseurl, own_key, paths, hashes, session=None): def get_curpos(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/frontend/currentposition", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"position"] @@ -222,8 +220,7 @@ def frontend_verify_entries(node, baseurl, own_key, paths, size): arguments = {"verify_to": size} result = http_request(baseurl + "plop/v1/frontend/verify-entries", json.dumps(arguments), - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"verified"] @@ -236,8 +233,7 @@ def frontend_verify_entries(node, baseurl, own_key, paths, size): def get_verifiedsize(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/merge/verifiedsize", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"size"] @@ -252,7 +248,7 @@ def sendlog(node, baseurl, own_key, paths, submission): try: result = http_request(baseurl + "plop/v1/frontend/sendlog", json.dumps(submission), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: sendlog", e.response @@ -271,7 +267,7 @@ def backup_sendlog(node, baseurl, own_key, paths, submission): try: result = http_request(baseurl + "plop/v1/merge/sendlog", json.dumps(submission), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: backup_sendlog", e.response @@ -296,7 +292,7 @@ def sendentries(node, baseurl, own_key, paths, entries, session=None): result = http_request( baseurl + "plop/v1/frontend/sendentry", json.dumps(json_entries), - key=own_key, verifynode=node, publickeydir=paths["publickeys"], + key=own_key, verifynode=node, session=session) return json.loads(result) except requests.exceptions.HTTPError, e: @@ -320,7 +316,7 @@ def sendentries_merge(node, baseurl, own_key, paths, entries, session=None): result = http_request( baseurl + "plop/v1/merge/sendentry", json.dumps(json_entries), - key=own_key, verifynode=node, publickeydir=paths["publickeys"], + key=own_key, verifynode=node, session=session) return json.loads(result) except requests.exceptions.HTTPError, e: @@ -342,7 +338,7 @@ def publish_sth(node, baseurl, own_key, paths, submission): try: result = http_request(baseurl + "plop/v1/frontend/publish-sth", json.dumps(submission), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: publish-sth", e.response @@ -360,7 +356,7 @@ def verifyroot(node, baseurl, own_key, paths, treesize): try: result = http_request(baseurl + "plop/v1/merge/verifyroot", json.dumps({"tree_size":treesize}), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: verifyroot", e.response @@ -378,7 +374,7 @@ def setverifiedsize(node, baseurl, own_key, paths, treesize): try: result = http_request(baseurl + "plop/v1/merge/setverifiedsize", json.dumps({"size":treesize}), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: setverifiedsize", e.response @@ -395,8 +391,7 @@ def setverifiedsize(node, baseurl, own_key, paths, treesize): def get_missingentries(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/frontend/missingentries", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"entries"] @@ -409,8 +404,7 @@ def get_missingentries(node, baseurl, own_key, paths): def get_missingentriesforbackup(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/merge/missingentries", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"entries"] @@ -439,6 +433,8 @@ def parse_args(): config = yaml.load(open(args.config)) localconfig = yaml.load(open(args.localconfig)) + set_api_keys(config) + return (args, config, localconfig) def perm(dbtype, path): |