summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMagnus Ahltorp <map@kth.se>2017-01-20 00:32:45 +0100
committerMagnus Ahltorp <map@kth.se>2017-01-20 00:32:45 +0100
commitd94247cb9f7746f75b176cbed0a32e9e902e7e7d (patch)
tree2aefb213f8c66c4bf464f58c62b81994f751dcab
parent1fecbcbcc5b7ae8453e68f3df6f25a140aa6df72 (diff)
API keys are now provided in the config file.
Also added CA cert verification for internal TLS connections.
-rw-r--r--test/Makefile1
-rw-r--r--test/catlfish-test.cfg.in (renamed from test/catlfish-test.cfg)0
-rwxr-xr-xtest/scripts/light-system-test-prepare.sh29
-rw-r--r--tools/certtools.py24
-rwxr-xr-xtools/compileconfig.py14
-rw-r--r--tools/mergetools.py38
6 files changed, 65 insertions, 41 deletions
diff --git a/test/Makefile b/test/Makefile
index c92c30d..f311208 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -11,6 +11,7 @@ tests-wait:
sleep 5
tests-makemk:
+ cat $(PREFIX)/test/catlfish-test.cfg.in | sed 's/@[A-Z0-9-]*@//' > $(PREFIX)/test/catlfish-test.cfg
$(PREFIX)/tools/compileconfig.py --config=$(PREFIX)/test/catlfish-test.cfg --testshellvars=$(PREFIX)/test/test.shvars --machines 1
tests:
diff --git a/test/catlfish-test.cfg b/test/catlfish-test.cfg.in
index 39288c7..39288c7 100644
--- a/test/catlfish-test.cfg
+++ b/test/catlfish-test.cfg.in
diff --git a/test/scripts/light-system-test-prepare.sh b/test/scripts/light-system-test-prepare.sh
index 6f6dd07..df45d25 100755
--- a/test/scripts/light-system-test-prepare.sh
+++ b/test/scripts/light-system-test-prepare.sh
@@ -38,6 +38,7 @@ createcert () {
createca
createcert
+cafingerprint=$(openssl x509 -in httpsca/demoCA/cacert.pem -noout -sha256 -fingerprint | sed -e 's/.*=//' -e 's/://g')
mkdir keys
(cd keys ; ${top_srcdir}/tools/create-key.sh logkey)
openssl pkcs8 -topk8 -nocrypt -in keys/logkey-private.pem -out keys/logkey-private.pkcs8
@@ -48,6 +49,23 @@ touch mergedb-secondary/logorder
printf 0 > mergedb-secondary/verifiedsize
mkdir known_roots
cp ${top_srcdir}/tools/testcerts/roots/* known_roots
+mkdir privatekeys
+mkdir publickeys
+echo "apikeys:" > api-keys.cfg
+for node in ${NODES}; do \
+ (cd privatekeys ; ${top_srcdir}/tools/create-key.sh ${node})
+ apipk=$(grep -v '^-----' privatekeys/${node}.pem | tr '\n' ' ')
+ mkdir -p nodes/${node}/log
+ echo " - nodename: ${node}" >> api-keys.cfg
+ echo " publickey: ${apipk}" >> api-keys.cfg
+done
+
+logpk=$(grep -v '^-----' keys/logkey.pem | tr '\n' ' ')
+echo "logpublickey: ${logpk}" >> api-keys.cfg
+echo "cafingerprint: ${cafingerprint}" >> api-keys.cfg
+
+
+cat ${top_srcdir}/test/catlfish-test.cfg.in api-keys.cfg > ${top_srcdir}/test/catlfish-test.cfg
for machine in ${MACHINES}; do \
${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-${machine}.cfg
mkdir -p machine/machine-${machine}/db
@@ -55,16 +73,5 @@ for machine in ${MACHINES}; do \
done
${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge-2.cfg
${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-signing.cfg
-mkdir privatekeys
-mkdir publickeys
-for node in ${NODES}; do \
- (cd privatekeys ; ${top_srcdir}/tools/create-key.sh ${node}) ; \
- mv privatekeys/${node}.pem publickeys/ ; \
- mkdir -p nodes/${node}/log
-done
-(cd privatekeys ; ${top_srcdir}/tools/create-key.sh merge-1)
-mv privatekeys/merge-1.pem publickeys/
-(cd privatekeys ; ${top_srcdir}/tools/create-key.sh merge-2)
-mv privatekeys/merge-2.pem publickeys/
test -x ${SOFTHSM} && ${SOFTHSM} --init-token --slot=0 --label=mylabel --so-pin=ffff --pin=ffff || true
test -x ${SOFTHSM} && ${SOFTHSM} --import keys/logkey-private.pkcs8 --slot 0 --label mylabel --pin ffff --id 00 || true
diff --git a/tools/certtools.py b/tools/certtools.py
index 0009d5d..0ccbcad 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -94,6 +94,12 @@ def get_root_cert(issuer):
class sslparameters:
cafile = None
+class apikeys:
+ publickeys = {}
+
+def set_api_keys(config):
+ apikeys.publickeys = dict([(node["nodename"], base64.b64decode(node["publickey"])) for node in config["apikeys"]])
+
def create_ssl_context(cafile=None):
try:
sslparameters.cafile = cafile
@@ -256,32 +262,32 @@ def parse_auth_header(authheader):
options = dict([(e.partition("=")[0], e.partition("=")[2]) for e in rawoptions])
return (base64.b64decode(signature), options)
-def check_auth_header(authheader, expected_key, publickeydir, data, path):
+def check_auth_header(authheader, expected_key, data, path):
if expected_key == None:
return True
(signature, options) = parse_auth_header(authheader)
+ publickey = apikeys.publickeys[expected_key]
keyname = options.get("key")
if keyname != expected_key:
raise Exception("Response claimed to come from %s, expected %s" % (keyname, expected_key))
- publickey = get_public_key_from_file(publickeydir + "/" + keyname + ".pem")
vk = ecdsa.VerifyingKey.from_der(publickey)
vk.verify(signature, "%s\0%s\0%s" % ("REPLY", path, data), hashfunc=hashlib.sha256,
sigdecode=ecdsa.util.sigdecode_der)
return True
-def http_request(url, data=None, key=None, verifynode=None, publickeydir=".", params=None, session=None):
+def http_request(url, data=None, key=None, verifynode=None, params=None, session=None):
if session:
- return http_request_session(url, data=data, key=key, verifynode=verifynode, publickeydir=publickeydir, params=params, session=session)
+ return http_request_session(url, data=data, key=key, verifynode=verifynode, params=params, session=session)
else:
with requests.sessions.Session() as session:
- return http_request_session(url, data=data, key=key, verifynode=verifynode, publickeydir=publickeydir, params=params, session=session)
+ return http_request_session(url, data=data, key=key, verifynode=verifynode, params=params, session=session)
def chunk_generator(data, maxsize):
while len(data):
yield data[:maxsize]
data = data[maxsize:]
-def http_request_session(url, data=None, key=None, verifynode=None, publickeydir=".", params=None, session=None):
+def http_request_session(url, data=None, key=None, verifynode=None, params=None, session=None):
(keyname, keyfile) = key
privatekey = get_eckey_from_file(keyfile)
sk = ecdsa.SigningKey.from_der(privatekey)
@@ -312,7 +318,7 @@ def http_request_session(url, data=None, key=None, verifynode=None, publickeydir
result.raise_for_status()
authheader = result.headers.get('X-Catlfish-Auth')
data = result.text
- check_auth_header(authheader, verifynode, publickeydir, data, url_to_sign)
+ check_auth_header(authheader, verifynode, data, url_to_sign)
return data
def get_signature(baseurl, data, key=None):
@@ -431,8 +437,10 @@ def timing_point(timer_dict=None, name=None):
starttime = timer_dict["lasttime"]
stoptime = t
deltatime = stoptime - starttime
- timer_dict["deltatimes"].append((name, deltatime.seconds * 1000000 + deltatime.microseconds))
+ microseconds = deltatime.seconds * 1000000 + deltatime.microseconds
+ timer_dict["deltatimes"].append((name, microseconds))
timer_dict["lasttime"] = t
+ #print name, microseconds/1000000.0
return None
else:
timer_dict = {"deltatimes":[], "lasttime":t}
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 1fa352e..bbc2277 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -7,10 +7,14 @@ import argparse
import sys
import yaml
import re
+import base64
class Symbol(str):
pass
+class Binary(str):
+ pass
+
clean_string = re.compile(r'^[-.:_/A-Za-z0-9 ]*$')
clean_symbol = re.compile(r'^[_A-Za-z0-9]*$')
@@ -34,6 +38,8 @@ def gen_erlang(term, level=1):
separator = ",\n" + indent
if isinstance(term, Symbol):
return quote_erlang_symbol(term)
+ elif isinstance(term, Binary):
+ return "<<" + ",".join([str(ord(c)) for c in term]) + ">>"
elif isinstance(term, basestring):
return quote_erlang_string(term)
elif isinstance(term, int):
@@ -194,9 +200,13 @@ def parse_ratelimit((type, description)):
print >>sys.stderr, "%s: Only one ratelimit expression supported right now" % (type,)
return (Symbol(type), descriptions)
+def api_keys(config):
+ return [(node["nodename"], Binary(base64.b64decode(node["publickey"]))) for node in config["apikeys"]]
+
def gen_config(nodename, config, localconfig):
print "generating config for", nodename
paths = localconfig["paths"]
+ apikeys = api_keys(config)
bind_addresses = {
"frontend": localconfig.get("frontendaddresses", {}).get(nodename),
"storage": localconfig.get("storageaddresses", {}).get(nodename),
@@ -231,6 +241,7 @@ def gen_config(nodename, config, localconfig):
(Symbol("https_certfile"), paths["https_certfile"]),
(Symbol("https_keyfile"), paths["https_keyfile"]),
(Symbol("https_cacertfile"), paths["https_cacertfile"]),
+ (Symbol("https_cacert_fingerprint"), Binary(base64.b16decode(config["cafingerprint"]))),
]
catlfishconfig.append((Symbol("mmd"), config["mmd"]))
@@ -333,7 +344,7 @@ def gen_config(nodename, config, localconfig):
print >>sys.stderr, "Neither logprivatekey nor hsm configured for signing node", nodename
sys.exit(1)
plopconfig += [
- (Symbol("log_public_key"), paths["logpublickey"]),
+ (Symbol("log_public_key"), Binary(base64.b64decode(config["logpublickey"]))),
(Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))),
]
if "frontendnodes" in nodetype:
@@ -341,6 +352,7 @@ def gen_config(nodename, config, localconfig):
plopconfig += [
(Symbol("allowed_clients"), list(allowed_clients)),
(Symbol("allowed_servers"), list(allowed_servers)),
+ (Symbol("apikeys"), apikeys),
]
erlangconfig = [
diff --git a/tools/mergetools.py b/tools/mergetools.py
index 94901a9..19d16ca 100644
--- a/tools/mergetools.py
+++ b/tools/mergetools.py
@@ -14,7 +14,7 @@ try:
import permdb
except ImportError:
pass
-from certtools import get_leaf_hash, http_request, get_leaf_hash
+from certtools import get_leaf_hash, http_request, get_leaf_hash, set_api_keys
def parselogrow(row):
return base64.b16decode(row, casefold=True)
@@ -167,8 +167,7 @@ def fsync_logorder(logorderfile):
def get_new_entries(node, baseurl, own_key, paths):
try:
result = http_request(baseurl + "plop/v1/storage/fetchnewentries",
- key=own_key, verifynode=node,
- publickeydir=paths["publickeys"])
+ key=own_key, verifynode=node)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return [base64.b64decode(entry) for \
@@ -185,7 +184,7 @@ def get_entries(node, baseurl, own_key, paths, hashes, session=None):
result = http_request(baseurl + "plop/v1/storage/getentry",
params=params,
key=own_key, verifynode=node,
- publickeydir=paths["publickeys"], session=session)
+ session=session)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
entries = dict([(base64.b64decode(entry["hash"]),
@@ -203,8 +202,7 @@ def get_entries(node, baseurl, own_key, paths, hashes, session=None):
def get_curpos(node, baseurl, own_key, paths):
try:
result = http_request(baseurl + "plop/v1/frontend/currentposition",
- key=own_key, verifynode=node,
- publickeydir=paths["publickeys"])
+ key=own_key, verifynode=node)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return parsed_result[u"position"]
@@ -222,8 +220,7 @@ def frontend_verify_entries(node, baseurl, own_key, paths, size):
arguments = {"verify_to": size}
result = http_request(baseurl + "plop/v1/frontend/verify-entries",
json.dumps(arguments),
- key=own_key, verifynode=node,
- publickeydir=paths["publickeys"])
+ key=own_key, verifynode=node)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return parsed_result[u"verified"]
@@ -236,8 +233,7 @@ def frontend_verify_entries(node, baseurl, own_key, paths, size):
def get_verifiedsize(node, baseurl, own_key, paths):
try:
result = http_request(baseurl + "plop/v1/merge/verifiedsize",
- key=own_key, verifynode=node,
- publickeydir=paths["publickeys"])
+ key=own_key, verifynode=node)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return parsed_result[u"size"]
@@ -252,7 +248,7 @@ def sendlog(node, baseurl, own_key, paths, submission):
try:
result = http_request(baseurl + "plop/v1/frontend/sendlog",
json.dumps(submission), key=own_key,
- verifynode=node, publickeydir=paths["publickeys"])
+ verifynode=node)
return json.loads(result)
except requests.exceptions.HTTPError, e:
print >>sys.stderr, "ERROR: sendlog", e.response
@@ -271,7 +267,7 @@ def backup_sendlog(node, baseurl, own_key, paths, submission):
try:
result = http_request(baseurl + "plop/v1/merge/sendlog",
json.dumps(submission), key=own_key,
- verifynode=node, publickeydir=paths["publickeys"])
+ verifynode=node)
return json.loads(result)
except requests.exceptions.HTTPError, e:
print >>sys.stderr, "ERROR: backup_sendlog", e.response
@@ -296,7 +292,7 @@ def sendentries(node, baseurl, own_key, paths, entries, session=None):
result = http_request(
baseurl + "plop/v1/frontend/sendentry",
json.dumps(json_entries),
- key=own_key, verifynode=node, publickeydir=paths["publickeys"],
+ key=own_key, verifynode=node,
session=session)
return json.loads(result)
except requests.exceptions.HTTPError, e:
@@ -320,7 +316,7 @@ def sendentries_merge(node, baseurl, own_key, paths, entries, session=None):
result = http_request(
baseurl + "plop/v1/merge/sendentry",
json.dumps(json_entries),
- key=own_key, verifynode=node, publickeydir=paths["publickeys"],
+ key=own_key, verifynode=node,
session=session)
return json.loads(result)
except requests.exceptions.HTTPError, e:
@@ -342,7 +338,7 @@ def publish_sth(node, baseurl, own_key, paths, submission):
try:
result = http_request(baseurl + "plop/v1/frontend/publish-sth",
json.dumps(submission), key=own_key,
- verifynode=node, publickeydir=paths["publickeys"])
+ verifynode=node)
return json.loads(result)
except requests.exceptions.HTTPError, e:
print >>sys.stderr, "ERROR: publish-sth", e.response
@@ -360,7 +356,7 @@ def verifyroot(node, baseurl, own_key, paths, treesize):
try:
result = http_request(baseurl + "plop/v1/merge/verifyroot",
json.dumps({"tree_size":treesize}), key=own_key,
- verifynode=node, publickeydir=paths["publickeys"])
+ verifynode=node)
return json.loads(result)
except requests.exceptions.HTTPError, e:
print >>sys.stderr, "ERROR: verifyroot", e.response
@@ -378,7 +374,7 @@ def setverifiedsize(node, baseurl, own_key, paths, treesize):
try:
result = http_request(baseurl + "plop/v1/merge/setverifiedsize",
json.dumps({"size":treesize}), key=own_key,
- verifynode=node, publickeydir=paths["publickeys"])
+ verifynode=node)
return json.loads(result)
except requests.exceptions.HTTPError, e:
print >>sys.stderr, "ERROR: setverifiedsize", e.response
@@ -395,8 +391,7 @@ def setverifiedsize(node, baseurl, own_key, paths, treesize):
def get_missingentries(node, baseurl, own_key, paths):
try:
result = http_request(baseurl + "plop/v1/frontend/missingentries",
- key=own_key, verifynode=node,
- publickeydir=paths["publickeys"])
+ key=own_key, verifynode=node)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return parsed_result[u"entries"]
@@ -409,8 +404,7 @@ def get_missingentries(node, baseurl, own_key, paths):
def get_missingentriesforbackup(node, baseurl, own_key, paths):
try:
result = http_request(baseurl + "plop/v1/merge/missingentries",
- key=own_key, verifynode=node,
- publickeydir=paths["publickeys"])
+ key=own_key, verifynode=node)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return parsed_result[u"entries"]
@@ -439,6 +433,8 @@ def parse_args():
config = yaml.load(open(args.config))
localconfig = yaml.load(open(args.localconfig))
+ set_api_keys(config)
+
return (args, config, localconfig)
def perm(dbtype, path):