diff options
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | catlfish.config | 5 | ||||
| -rw-r--r-- | httpd_props.conf | 2 | ||||
| -rw-r--r-- | storage_node.config | 19 | ||||
| -rw-r--r-- | storage_node_httpd.conf | 21 | ||||
| -rwxr-xr-x | tools/merge.py | 133 | ||||
| -rwxr-xr-x | tools/testcase1.py | 13 |
7 files changed, 196 insertions, 2 deletions
@@ -10,8 +10,13 @@ release: ln -s ../../plop/test .) cp httpd_props.conf rel cp catlfish.config rel + cp storage_node.config rel + cp storage_node_httpd.conf rel mkdir rel/catlfish mkdir rel/db + mkdir rel/mergedb + mkdir rel/mergedb/chains + touch rel/mergedb/logorder printf "0" > rel/db/treesize cp -r webroot rel/catlfish test -d rel/catlfish/webroot/log || mkdir rel/catlfish/webroot/log diff --git a/catlfish.config b/catlfish.config index 75d00fa..672f997 100644 --- a/catlfish.config +++ b/catlfish.config @@ -21,4 +21,7 @@ {index_path, "db/index"}, {entryhash_root_path, "db/entryhash/"}, {treesize_path, "db/treesize"}, - {indexforhash_root_path, "db/certindex/"}]}]. + {indexforhash_root_path, "db/certindex/"}, + %{storage_nodes, ["https://127.0.0.1:8081/ct/storage/"]}, + {storage_nodes_quorum, 1} + ]}]. diff --git a/httpd_props.conf b/httpd_props.conf index ae4c7be..9ea7b30 100644 --- a/httpd_props.conf +++ b/httpd_props.conf @@ -8,7 +8,7 @@ {document_root, "catlfish/webroot/docroot"}, {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head, mod_log, mod_disk_log]}, - {erl_script_alias, {"/ct", [v1]}}, + {erl_script_alias, {"/ct", [v1, frontend]}}, {erl_script_nocache, true}, {error_log, "log/error"}, {security_log, "log/security"}, diff --git a/storage_node.config b/storage_node.config new file mode 100644 index 0000000..47a1326 --- /dev/null +++ b/storage_node.config @@ -0,0 +1,19 @@ +%% catlfish configuration file (-*- erlang -*-) +%% Start like this: +%% $ erl -boot start_sasl -config catlfish -run inets +[{sasl, + [{sasl_error_logger, false}, + {errlog_type, error}, + {error_logger_mf_dir, "log"}, + {error_logger_mf_maxbytes, 10485760}, % 10 MB + {error_logger_mf_maxfiles, 10}]}, + {inets, + [{services, + [{httpd, [{proplist_file, "storage_node_httpd.conf"}]}]}]}, + {plop, + [{entry_root_path, "db/certentries/"}, + {index_path, "db/index"}, + {newentries_path, "db/newentries"}, + {entryhash_root_path, "db/entryhash/"}, + {treesize_path, "db/treesize"}, + {indexforhash_root_path, "db/certindex/"}]}]. diff --git a/storage_node_httpd.conf b/storage_node_httpd.conf new file mode 100644 index 0000000..2f271f8 --- /dev/null +++ b/storage_node_httpd.conf @@ -0,0 +1,21 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. +[ + {port, 8081}, + {bind_address, {127, 0, 0, 1}}, + {server_name, "flimsy"}, + {server_root, "catlfish/webroot"}, + {document_root, "catlfish/webroot/docroot"}, + {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head, + mod_log, mod_disk_log]}, + {erl_script_alias, {"/ct", [storage]}}, + {erl_script_nocache, true}, + {error_log, "log/error_storage"}, + {security_log, "log/security_storage"}, + {transfer_log, "log/transfer_storage"}, + {socket_type, + {essl, % See ssl(3erl) for SSL options. + [{certfile, "catlfish/webroot/certs/webcert.pem"}, + {keyfile, "catlfish/webroot/keys/webkey.pem"}, + {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}} +]. diff --git a/tools/merge.py b/tools/merge.py new file mode 100755 index 0000000..7120d04 --- /dev/null +++ b/tools/merge.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Kungliga Tekniska Högskolan +# (KTH Royal Institute of Technology, Stockholm, Sweden). +# See LICENSE for licensing information. + +import json +import base64 +import urllib +import urllib2 +import sys + +frontendnodes = ["https://127.0.0.1:8080/"] +storagenodes = ["https://127.0.0.1:8081/"] + +chainsdir = "../rel/mergedb/chains" +logorderfile = "../rel/mergedb/logorder" + +def parselogrow(row): + return base64.b16decode(row) + +def get_logorder(): + f = open(logorderfile, "r") + return [parselogrow(row.rstrip()) for row in f] + +def write_chain(key, value): + f = open(chainsdir + "/" + base64.b16encode(key), "w") + f.write(value) + f.close() + +def read_chain(key): + f = open(chainsdir + "/" + base64.b16encode(key), "r") + value = f.read() + f.close() + return value + +def add_to_logorder(key): + f = open(logorderfile, "a") + f.write(base64.b16encode(key) + "\n") + f.close() + +def get_new_entries(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/storage/fetchnewentries").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"entries"] + print "ERROR: fetchnewentries", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: fetchnewentries", e.read() + sys.exit(1) + +def get_curpos(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/currentposition").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"position"] + print "ERROR: currentposition", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: currentposition", e.read() + sys.exit(1) + +def sendlog(baseurl, submission): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendlog", + json.dumps(submission)).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendlog", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e + +def sendsth(baseurl, submission): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendsth", + json.dumps(submission)).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendsth", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e + +def get_missingentries(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/missingentries").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"entries"] + print "ERROR: missingentries", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: missingentries", e.read() + sys.exit(1) + + +logorder = get_logorder() +certsinlog = set(logorder) + +new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] + +for new_entry in new_entries: + hash = base64.b64decode(new_entry["hash"]) + entry = base64.b64decode(new_entry["entry"]) + if hash not in certsinlog: + write_chain(hash, entry) + add_to_logorder(hash) + logorder.append(hash) + certsinlog.add(hash) + print "added", base64.b16encode(hash) + +for frontendnode in frontendnodes: + curpos = get_curpos(frontendnode) + entries = [base64.b64encode(entry) for entry in logorder[curpos:]] + sendlog(frontendnode, {"start": curpos, "hashes": entries}) + missingentries = get_missingentries(frontendnode) + print "missing entries:", missingentries + # XXX: no test case for missing entries yet, waiting to implement + sendsth(frontendnode, {"tree_size": len(logorder)}) diff --git a/tools/testcase1.py b/tools/testcase1.py index eab6c6f..2d5e0e8 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Copyright (c) 2014, NORDUnet A/S. # See LICENSE for licensing information. @@ -125,12 +126,16 @@ testgroup("cert1") result1 = do_add_chain(cc1) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(1) result2 = do_add_chain(cc1) assert_equal(result2["timestamp"], result1["timestamp"], "timestamp") +subprocess.call(["./merge.py"]) + print_and_check_tree_size(1) # TODO: add invalid cert and check that it generates an error @@ -142,6 +147,8 @@ testgroup("cert2") result3 = do_add_chain(cc2) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(2) get_and_validate_proof(result1["timestamp"], cc1, 0, 1) @@ -151,6 +158,8 @@ testgroup("cert3") result4 = do_add_chain(cc3) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(3) get_and_validate_proof(result1["timestamp"], cc1, 0, 2) @@ -161,6 +170,8 @@ testgroup("cert4") result5 = do_add_chain(cc4) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(4) get_and_validate_proof(result1["timestamp"], cc1, 0, 2) @@ -172,6 +183,8 @@ testgroup("cert5") result6 = do_add_chain(cc5) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(5) get_and_validate_proof(result1["timestamp"], cc1, 0, 3) |