From 4325a43d3d1476cc601b631620df89340ec4c7e2 Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Mon, 20 Oct 2014 14:33:41 +0200
Subject: Added external merging support
---
Makefile | 5 ++
catlfish.config | 5 +-
httpd_props.conf | 2 +-
storage_node.config | 19 +++++++
storage_node_httpd.conf | 21 ++++++++
tools/merge.py | 133 ++++++++++++++++++++++++++++++++++++++++++++++++
tools/testcase1.py | 13 +++++
7 files changed, 196 insertions(+), 2 deletions(-)
create mode 100644 storage_node.config
create mode 100644 storage_node_httpd.conf
create mode 100755 tools/merge.py
diff --git a/Makefile b/Makefile
index 78f8a6b..aa96290 100644
--- a/Makefile
+++ b/Makefile
@@ -10,8 +10,13 @@ release:
ln -s ../../plop/test .)
cp httpd_props.conf rel
cp catlfish.config rel
+ cp storage_node.config rel
+ cp storage_node_httpd.conf rel
mkdir rel/catlfish
mkdir rel/db
+ mkdir rel/mergedb
+ mkdir rel/mergedb/chains
+ touch rel/mergedb/logorder
printf "0" > rel/db/treesize
cp -r webroot rel/catlfish
test -d rel/catlfish/webroot/log || mkdir rel/catlfish/webroot/log
diff --git a/catlfish.config b/catlfish.config
index 75d00fa..672f997 100644
--- a/catlfish.config
+++ b/catlfish.config
@@ -21,4 +21,7 @@
{index_path, "db/index"},
{entryhash_root_path, "db/entryhash/"},
{treesize_path, "db/treesize"},
- {indexforhash_root_path, "db/certindex/"}]}].
+ {indexforhash_root_path, "db/certindex/"},
+ %{storage_nodes, ["https://127.0.0.1:8081/ct/storage/"]},
+ {storage_nodes_quorum, 1}
+ ]}].
diff --git a/httpd_props.conf b/httpd_props.conf
index ae4c7be..9ea7b30 100644
--- a/httpd_props.conf
+++ b/httpd_props.conf
@@ -8,7 +8,7 @@
{document_root, "catlfish/webroot/docroot"},
{modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head,
mod_log, mod_disk_log]},
- {erl_script_alias, {"/ct", [v1]}},
+ {erl_script_alias, {"/ct", [v1, frontend]}},
{erl_script_nocache, true},
{error_log, "log/error"},
{security_log, "log/security"},
diff --git a/storage_node.config b/storage_node.config
new file mode 100644
index 0000000..47a1326
--- /dev/null
+++ b/storage_node.config
@@ -0,0 +1,19 @@
+%% catlfish configuration file (-*- erlang -*-)
+%% Start like this:
+%% $ erl -boot start_sasl -config catlfish -run inets
+[{sasl,
+ [{sasl_error_logger, false},
+ {errlog_type, error},
+ {error_logger_mf_dir, "log"},
+ {error_logger_mf_maxbytes, 10485760}, % 10 MB
+ {error_logger_mf_maxfiles, 10}]},
+ {inets,
+ [{services,
+ [{httpd, [{proplist_file, "storage_node_httpd.conf"}]}]}]},
+ {plop,
+ [{entry_root_path, "db/certentries/"},
+ {index_path, "db/index"},
+ {newentries_path, "db/newentries"},
+ {entryhash_root_path, "db/entryhash/"},
+ {treesize_path, "db/treesize"},
+ {indexforhash_root_path, "db/certindex/"}]}].
diff --git a/storage_node_httpd.conf b/storage_node_httpd.conf
new file mode 100644
index 0000000..2f271f8
--- /dev/null
+++ b/storage_node_httpd.conf
@@ -0,0 +1,21 @@
+%%% Copyright (c) 2014, NORDUnet A/S.
+%%% See LICENSE for licensing information.
+[
+ {port, 8081},
+ {bind_address, {127, 0, 0, 1}},
+ {server_name, "flimsy"},
+ {server_root, "catlfish/webroot"},
+ {document_root, "catlfish/webroot/docroot"},
+ {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head,
+ mod_log, mod_disk_log]},
+ {erl_script_alias, {"/ct", [storage]}},
+ {erl_script_nocache, true},
+ {error_log, "log/error_storage"},
+ {security_log, "log/security_storage"},
+ {transfer_log, "log/transfer_storage"},
+ {socket_type,
+ {essl, % See ssl(3erl) for SSL options.
+ [{certfile, "catlfish/webroot/certs/webcert.pem"},
+ {keyfile, "catlfish/webroot/keys/webkey.pem"},
+ {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}}
+].
diff --git a/tools/merge.py b/tools/merge.py
new file mode 100755
index 0000000..7120d04
--- /dev/null
+++ b/tools/merge.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2014 Kungliga Tekniska Högskolan
+# (KTH Royal Institute of Technology, Stockholm, Sweden).
+# See LICENSE for licensing information.
+
+import json
+import base64
+import urllib
+import urllib2
+import sys
+
+frontendnodes = ["https://127.0.0.1:8080/"]
+storagenodes = ["https://127.0.0.1:8081/"]
+
+chainsdir = "../rel/mergedb/chains"
+logorderfile = "../rel/mergedb/logorder"
+
+def parselogrow(row):
+ return base64.b16decode(row)
+
+def get_logorder():
+ f = open(logorderfile, "r")
+ return [parselogrow(row.rstrip()) for row in f]
+
+def write_chain(key, value):
+ f = open(chainsdir + "/" + base64.b16encode(key), "w")
+ f.write(value)
+ f.close()
+
+def read_chain(key):
+ f = open(chainsdir + "/" + base64.b16encode(key), "r")
+ value = f.read()
+ f.close()
+ return value
+
+def add_to_logorder(key):
+ f = open(logorderfile, "a")
+ f.write(base64.b16encode(key) + "\n")
+ f.close()
+
+def get_new_entries(baseurl):
+ try:
+ result = urllib2.urlopen(baseurl + "ct/storage/fetchnewentries").read()
+ parsed_result = json.loads(result)
+ if parsed_result.get(u"result") == u"ok":
+ return parsed_result[u"entries"]
+ print "ERROR: fetchnewentries", parsed_result
+ sys.exit(1)
+ except urllib2.HTTPError, e:
+ print "ERROR: fetchnewentries", e.read()
+ sys.exit(1)
+
+def get_curpos(baseurl):
+ try:
+ result = urllib2.urlopen(baseurl + "ct/frontend/currentposition").read()
+ parsed_result = json.loads(result)
+ if parsed_result.get(u"result") == u"ok":
+ return parsed_result[u"position"]
+ print "ERROR: currentposition", parsed_result
+ sys.exit(1)
+ except urllib2.HTTPError, e:
+ print "ERROR: currentposition", e.read()
+ sys.exit(1)
+
+def sendlog(baseurl, submission):
+ try:
+ result = urllib2.urlopen(baseurl + "ct/frontend/sendlog",
+ json.dumps(submission)).read()
+ return json.loads(result)
+ except urllib2.HTTPError, e:
+ print "ERROR: sendlog", e.read()
+ sys.exit(1)
+ except ValueError, e:
+ print "==== FAILED REQUEST ===="
+ print submission
+ print "======= RESPONSE ======="
+ print result
+ print "========================"
+ raise e
+
+def sendsth(baseurl, submission):
+ try:
+ result = urllib2.urlopen(baseurl + "ct/frontend/sendsth",
+ json.dumps(submission)).read()
+ return json.loads(result)
+ except urllib2.HTTPError, e:
+ print "ERROR: sendsth", e.read()
+ sys.exit(1)
+ except ValueError, e:
+ print "==== FAILED REQUEST ===="
+ print submission
+ print "======= RESPONSE ======="
+ print result
+ print "========================"
+ raise e
+
+def get_missingentries(baseurl):
+ try:
+ result = urllib2.urlopen(baseurl + "ct/frontend/missingentries").read()
+ parsed_result = json.loads(result)
+ if parsed_result.get(u"result") == u"ok":
+ return parsed_result[u"entries"]
+ print "ERROR: missingentries", parsed_result
+ sys.exit(1)
+ except urllib2.HTTPError, e:
+ print "ERROR: missingentries", e.read()
+ sys.exit(1)
+
+
+logorder = get_logorder()
+certsinlog = set(logorder)
+
+new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)]
+
+for new_entry in new_entries:
+ hash = base64.b64decode(new_entry["hash"])
+ entry = base64.b64decode(new_entry["entry"])
+ if hash not in certsinlog:
+ write_chain(hash, entry)
+ add_to_logorder(hash)
+ logorder.append(hash)
+ certsinlog.add(hash)
+ print "added", base64.b16encode(hash)
+
+for frontendnode in frontendnodes:
+ curpos = get_curpos(frontendnode)
+ entries = [base64.b64encode(entry) for entry in logorder[curpos:]]
+ sendlog(frontendnode, {"start": curpos, "hashes": entries})
+ missingentries = get_missingentries(frontendnode)
+ print "missing entries:", missingentries
+ # XXX: no test case for missing entries yet, waiting to implement
+ sendsth(frontendnode, {"tree_size": len(logorder)})
diff --git a/tools/testcase1.py b/tools/testcase1.py
index eab6c6f..2d5e0e8 100755
--- a/tools/testcase1.py
+++ b/tools/testcase1.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
# Copyright (c) 2014, NORDUnet A/S.
# See LICENSE for licensing information.
@@ -125,12 +126,16 @@ testgroup("cert1")
result1 = do_add_chain(cc1)
+subprocess.call(["./merge.py"])
+
print_and_check_tree_size(1)
result2 = do_add_chain(cc1)
assert_equal(result2["timestamp"], result1["timestamp"], "timestamp")
+subprocess.call(["./merge.py"])
+
print_and_check_tree_size(1)
# TODO: add invalid cert and check that it generates an error
@@ -142,6 +147,8 @@ testgroup("cert2")
result3 = do_add_chain(cc2)
+subprocess.call(["./merge.py"])
+
print_and_check_tree_size(2)
get_and_validate_proof(result1["timestamp"], cc1, 0, 1)
@@ -151,6 +158,8 @@ testgroup("cert3")
result4 = do_add_chain(cc3)
+subprocess.call(["./merge.py"])
+
print_and_check_tree_size(3)
get_and_validate_proof(result1["timestamp"], cc1, 0, 2)
@@ -161,6 +170,8 @@ testgroup("cert4")
result5 = do_add_chain(cc4)
+subprocess.call(["./merge.py"])
+
print_and_check_tree_size(4)
get_and_validate_proof(result1["timestamp"], cc1, 0, 2)
@@ -172,6 +183,8 @@ testgroup("cert5")
result6 = do_add_chain(cc5)
+subprocess.call(["./merge.py"])
+
print_and_check_tree_size(5)
get_and_validate_proof(result1["timestamp"], cc1, 0, 3)
--
cgit v1.1