From d94247cb9f7746f75b176cbed0a32e9e902e7e7d Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 20 Jan 2017 00:32:45 +0100 Subject: API keys are now provided in the config file. Also added CA cert verification for internal TLS connections. --- tools/certtools.py | 24 ++++++++++++++++-------- tools/compileconfig.py | 14 +++++++++++++- tools/mergetools.py | 38 +++++++++++++++++--------------------- 3 files changed, 46 insertions(+), 30 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 0009d5d..0ccbcad 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -94,6 +94,12 @@ def get_root_cert(issuer): class sslparameters: cafile = None +class apikeys: + publickeys = {} + +def set_api_keys(config): + apikeys.publickeys = dict([(node["nodename"], base64.b64decode(node["publickey"])) for node in config["apikeys"]]) + def create_ssl_context(cafile=None): try: sslparameters.cafile = cafile @@ -256,32 +262,32 @@ def parse_auth_header(authheader): options = dict([(e.partition("=")[0], e.partition("=")[2]) for e in rawoptions]) return (base64.b64decode(signature), options) -def check_auth_header(authheader, expected_key, publickeydir, data, path): +def check_auth_header(authheader, expected_key, data, path): if expected_key == None: return True (signature, options) = parse_auth_header(authheader) + publickey = apikeys.publickeys[expected_key] keyname = options.get("key") if keyname != expected_key: raise Exception("Response claimed to come from %s, expected %s" % (keyname, expected_key)) - publickey = get_public_key_from_file(publickeydir + "/" + keyname + ".pem") vk = ecdsa.VerifyingKey.from_der(publickey) vk.verify(signature, "%s\0%s\0%s" % ("REPLY", path, data), hashfunc=hashlib.sha256, sigdecode=ecdsa.util.sigdecode_der) return True -def http_request(url, data=None, key=None, verifynode=None, publickeydir=".", params=None, session=None): +def http_request(url, data=None, key=None, verifynode=None, params=None, session=None): if session: - return http_request_session(url, data=data, key=key, verifynode=verifynode, publickeydir=publickeydir, params=params, session=session) + return http_request_session(url, data=data, key=key, verifynode=verifynode, params=params, session=session) else: with requests.sessions.Session() as session: - return http_request_session(url, data=data, key=key, verifynode=verifynode, publickeydir=publickeydir, params=params, session=session) + return http_request_session(url, data=data, key=key, verifynode=verifynode, params=params, session=session) def chunk_generator(data, maxsize): while len(data): yield data[:maxsize] data = data[maxsize:] -def http_request_session(url, data=None, key=None, verifynode=None, publickeydir=".", params=None, session=None): +def http_request_session(url, data=None, key=None, verifynode=None, params=None, session=None): (keyname, keyfile) = key privatekey = get_eckey_from_file(keyfile) sk = ecdsa.SigningKey.from_der(privatekey) @@ -312,7 +318,7 @@ def http_request_session(url, data=None, key=None, verifynode=None, publickeydir result.raise_for_status() authheader = result.headers.get('X-Catlfish-Auth') data = result.text - check_auth_header(authheader, verifynode, publickeydir, data, url_to_sign) + check_auth_header(authheader, verifynode, data, url_to_sign) return data def get_signature(baseurl, data, key=None): @@ -431,8 +437,10 @@ def timing_point(timer_dict=None, name=None): starttime = timer_dict["lasttime"] stoptime = t deltatime = stoptime - starttime - timer_dict["deltatimes"].append((name, deltatime.seconds * 1000000 + deltatime.microseconds)) + microseconds = deltatime.seconds * 1000000 + deltatime.microseconds + timer_dict["deltatimes"].append((name, microseconds)) timer_dict["lasttime"] = t + #print name, microseconds/1000000.0 return None else: timer_dict = {"deltatimes":[], "lasttime":t} diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 1fa352e..bbc2277 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -7,10 +7,14 @@ import argparse import sys import yaml import re +import base64 class Symbol(str): pass +class Binary(str): + pass + clean_string = re.compile(r'^[-.:_/A-Za-z0-9 ]*$') clean_symbol = re.compile(r'^[_A-Za-z0-9]*$') @@ -34,6 +38,8 @@ def gen_erlang(term, level=1): separator = ",\n" + indent if isinstance(term, Symbol): return quote_erlang_symbol(term) + elif isinstance(term, Binary): + return "<<" + ",".join([str(ord(c)) for c in term]) + ">>" elif isinstance(term, basestring): return quote_erlang_string(term) elif isinstance(term, int): @@ -194,9 +200,13 @@ def parse_ratelimit((type, description)): print >>sys.stderr, "%s: Only one ratelimit expression supported right now" % (type,) return (Symbol(type), descriptions) +def api_keys(config): + return [(node["nodename"], Binary(base64.b64decode(node["publickey"]))) for node in config["apikeys"]] + def gen_config(nodename, config, localconfig): print "generating config for", nodename paths = localconfig["paths"] + apikeys = api_keys(config) bind_addresses = { "frontend": localconfig.get("frontendaddresses", {}).get(nodename), "storage": localconfig.get("storageaddresses", {}).get(nodename), @@ -231,6 +241,7 @@ def gen_config(nodename, config, localconfig): (Symbol("https_certfile"), paths["https_certfile"]), (Symbol("https_keyfile"), paths["https_keyfile"]), (Symbol("https_cacertfile"), paths["https_cacertfile"]), + (Symbol("https_cacert_fingerprint"), Binary(base64.b16decode(config["cafingerprint"]))), ] catlfishconfig.append((Symbol("mmd"), config["mmd"])) @@ -333,7 +344,7 @@ def gen_config(nodename, config, localconfig): print >>sys.stderr, "Neither logprivatekey nor hsm configured for signing node", nodename sys.exit(1) plopconfig += [ - (Symbol("log_public_key"), paths["logpublickey"]), + (Symbol("log_public_key"), Binary(base64.b64decode(config["logpublickey"]))), (Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))), ] if "frontendnodes" in nodetype: @@ -341,6 +352,7 @@ def gen_config(nodename, config, localconfig): plopconfig += [ (Symbol("allowed_clients"), list(allowed_clients)), (Symbol("allowed_servers"), list(allowed_servers)), + (Symbol("apikeys"), apikeys), ] erlangconfig = [ diff --git a/tools/mergetools.py b/tools/mergetools.py index 94901a9..19d16ca 100644 --- a/tools/mergetools.py +++ b/tools/mergetools.py @@ -14,7 +14,7 @@ try: import permdb except ImportError: pass -from certtools import get_leaf_hash, http_request, get_leaf_hash +from certtools import get_leaf_hash, http_request, get_leaf_hash, set_api_keys def parselogrow(row): return base64.b16decode(row, casefold=True) @@ -167,8 +167,7 @@ def fsync_logorder(logorderfile): def get_new_entries(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/storage/fetchnewentries", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return [base64.b64decode(entry) for \ @@ -185,7 +184,7 @@ def get_entries(node, baseurl, own_key, paths, hashes, session=None): result = http_request(baseurl + "plop/v1/storage/getentry", params=params, key=own_key, verifynode=node, - publickeydir=paths["publickeys"], session=session) + session=session) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": entries = dict([(base64.b64decode(entry["hash"]), @@ -203,8 +202,7 @@ def get_entries(node, baseurl, own_key, paths, hashes, session=None): def get_curpos(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/frontend/currentposition", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"position"] @@ -222,8 +220,7 @@ def frontend_verify_entries(node, baseurl, own_key, paths, size): arguments = {"verify_to": size} result = http_request(baseurl + "plop/v1/frontend/verify-entries", json.dumps(arguments), - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"verified"] @@ -236,8 +233,7 @@ def frontend_verify_entries(node, baseurl, own_key, paths, size): def get_verifiedsize(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/merge/verifiedsize", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"size"] @@ -252,7 +248,7 @@ def sendlog(node, baseurl, own_key, paths, submission): try: result = http_request(baseurl + "plop/v1/frontend/sendlog", json.dumps(submission), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: sendlog", e.response @@ -271,7 +267,7 @@ def backup_sendlog(node, baseurl, own_key, paths, submission): try: result = http_request(baseurl + "plop/v1/merge/sendlog", json.dumps(submission), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: backup_sendlog", e.response @@ -296,7 +292,7 @@ def sendentries(node, baseurl, own_key, paths, entries, session=None): result = http_request( baseurl + "plop/v1/frontend/sendentry", json.dumps(json_entries), - key=own_key, verifynode=node, publickeydir=paths["publickeys"], + key=own_key, verifynode=node, session=session) return json.loads(result) except requests.exceptions.HTTPError, e: @@ -320,7 +316,7 @@ def sendentries_merge(node, baseurl, own_key, paths, entries, session=None): result = http_request( baseurl + "plop/v1/merge/sendentry", json.dumps(json_entries), - key=own_key, verifynode=node, publickeydir=paths["publickeys"], + key=own_key, verifynode=node, session=session) return json.loads(result) except requests.exceptions.HTTPError, e: @@ -342,7 +338,7 @@ def publish_sth(node, baseurl, own_key, paths, submission): try: result = http_request(baseurl + "plop/v1/frontend/publish-sth", json.dumps(submission), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: publish-sth", e.response @@ -360,7 +356,7 @@ def verifyroot(node, baseurl, own_key, paths, treesize): try: result = http_request(baseurl + "plop/v1/merge/verifyroot", json.dumps({"tree_size":treesize}), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: verifyroot", e.response @@ -378,7 +374,7 @@ def setverifiedsize(node, baseurl, own_key, paths, treesize): try: result = http_request(baseurl + "plop/v1/merge/setverifiedsize", json.dumps({"size":treesize}), key=own_key, - verifynode=node, publickeydir=paths["publickeys"]) + verifynode=node) return json.loads(result) except requests.exceptions.HTTPError, e: print >>sys.stderr, "ERROR: setverifiedsize", e.response @@ -395,8 +391,7 @@ def setverifiedsize(node, baseurl, own_key, paths, treesize): def get_missingentries(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/frontend/missingentries", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"entries"] @@ -409,8 +404,7 @@ def get_missingentries(node, baseurl, own_key, paths): def get_missingentriesforbackup(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/merge/missingentries", - key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + key=own_key, verifynode=node) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"entries"] @@ -439,6 +433,8 @@ def parse_args(): config = yaml.load(open(args.config)) localconfig = yaml.load(open(args.localconfig)) + set_api_keys(config) + return (args, config, localconfig) def perm(dbtype, path): -- cgit v1.1