From 194c9aa3b8c463fa487dc9ef7e172332a8d94d72 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 20 Oct 2014 14:33:41 +0200 Subject: Added external merging support --- tools/merge.py | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100755 tools/merge.py (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py new file mode 100755 index 0000000..7120d04 --- /dev/null +++ b/tools/merge.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Kungliga Tekniska Högskolan +# (KTH Royal Institute of Technology, Stockholm, Sweden). +# See LICENSE for licensing information. + +import json +import base64 +import urllib +import urllib2 +import sys + +frontendnodes = ["https://127.0.0.1:8080/"] +storagenodes = ["https://127.0.0.1:8081/"] + +chainsdir = "../rel/mergedb/chains" +logorderfile = "../rel/mergedb/logorder" + +def parselogrow(row): + return base64.b16decode(row) + +def get_logorder(): + f = open(logorderfile, "r") + return [parselogrow(row.rstrip()) for row in f] + +def write_chain(key, value): + f = open(chainsdir + "/" + base64.b16encode(key), "w") + f.write(value) + f.close() + +def read_chain(key): + f = open(chainsdir + "/" + base64.b16encode(key), "r") + value = f.read() + f.close() + return value + +def add_to_logorder(key): + f = open(logorderfile, "a") + f.write(base64.b16encode(key) + "\n") + f.close() + +def get_new_entries(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/storage/fetchnewentries").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"entries"] + print "ERROR: fetchnewentries", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: fetchnewentries", e.read() + sys.exit(1) + +def get_curpos(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/currentposition").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"position"] + print "ERROR: currentposition", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: currentposition", e.read() + sys.exit(1) + +def sendlog(baseurl, submission): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendlog", + json.dumps(submission)).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendlog", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e + +def sendsth(baseurl, submission): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendsth", + json.dumps(submission)).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendsth", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e + +def get_missingentries(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/missingentries").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"entries"] + print "ERROR: missingentries", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: missingentries", e.read() + sys.exit(1) + + +logorder = get_logorder() +certsinlog = set(logorder) + +new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] + +for new_entry in new_entries: + hash = base64.b64decode(new_entry["hash"]) + entry = base64.b64decode(new_entry["entry"]) + if hash not in certsinlog: + write_chain(hash, entry) + add_to_logorder(hash) + logorder.append(hash) + certsinlog.add(hash) + print "added", base64.b16encode(hash) + +for frontendnode in frontendnodes: + curpos = get_curpos(frontendnode) + entries = [base64.b64encode(entry) for entry in logorder[curpos:]] + sendlog(frontendnode, {"start": curpos, "hashes": entries}) + missingentries = get_missingentries(frontendnode) + print "missing entries:", missingentries + # XXX: no test case for missing entries yet, waiting to implement + sendsth(frontendnode, {"tree_size": len(logorder)}) -- cgit v1.1 From 574f96fba9fd01ec9725c5509f02ad13d8ab8793 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 24 Oct 2014 17:22:30 +0200 Subject: System tests for external merge --- tools/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py index 7120d04..14466ff 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -10,7 +10,7 @@ import urllib import urllib2 import sys -frontendnodes = ["https://127.0.0.1:8080/"] +frontendnodes = ["https://127.0.0.1:8082/"] storagenodes = ["https://127.0.0.1:8081/"] chainsdir = "../rel/mergedb/chains" -- cgit v1.1 From 7fd299ab23beee422789f679a35c9526c54fb1fb Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 14:31:20 +0100 Subject: Handle missing entries in merge --- tools/merge.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py index 14466ff..3aeecdf 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -79,6 +79,22 @@ def sendlog(baseurl, submission): print "========================" raise e +def sendentry(baseurl, entry, hash): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendentry", + json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)})).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendentry", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print hash + print "======= RESPONSE =======" + print result + print "========================" + raise e + def sendsth(baseurl, submission): try: result = urllib2.urlopen(baseurl + "ct/frontend/sendsth", @@ -113,6 +129,8 @@ certsinlog = set(logorder) new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] +print "adding entries" +added_entries = 0 for new_entry in new_entries: hash = base64.b64decode(new_entry["hash"]) entry = base64.b64decode(new_entry["entry"]) @@ -121,13 +139,19 @@ for new_entry in new_entries: add_to_logorder(hash) logorder.append(hash) certsinlog.add(hash) - print "added", base64.b16encode(hash) + added_entries += 1 +print "added", added_entries, "entries" for frontendnode in frontendnodes: + print "distributing for node", frontendnode curpos = get_curpos(frontendnode) + print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] sendlog(frontendnode, {"start": curpos, "hashes": entries}) + print "log sent" missingentries = get_missingentries(frontendnode) print "missing entries:", missingentries - # XXX: no test case for missing entries yet, waiting to implement + for missingentry in missingentries: + hash = base64.b64decode(missingentry) + sendentry(frontendnode, read_chain(hash), hash) sendsth(frontendnode, {"tree_size": len(logorder)}) -- cgit v1.1 From 89b8a59da38daf465a934f039cc7cf3efaf1468e Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 16:56:03 +0100 Subject: merge.py: send whole sth in sendsth call --- tools/merge.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py index 3aeecdf..7c6e357 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -9,7 +9,10 @@ import base64 import urllib import urllib2 import sys +import time +from certtools import build_merkle_tree, create_sth_signature, check_sth_signature +ctbaseurl = "https://127.0.0.1:8080/" frontendnodes = ["https://127.0.0.1:8082/"] storagenodes = ["https://127.0.0.1:8081/"] @@ -142,6 +145,28 @@ for new_entry in new_entries: added_entries += 1 print "added", added_entries, "entries" +tree = build_merkle_tree(logorder) +tree_size = len(logorder) +root_hash = tree[-1][0] +timestamp = int(time.time() * 1000) +privatekey = base64.decodestring( + "MHcCAQEEIMM/FjZ4FSzfENTTwGpTve6CP+IVr" + "Y7p8OKV634uJI/foAoGCCqGSM49AwEHoUQDQg" + "AE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9PMS5l" + "qoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJ" + "GolozZYmNHE2kQ==") + +tree_head_signature = create_sth_signature(tree_size, timestamp, + root_hash, privatekey) + +sth = {"tree_size": tree_size, "timestamp": timestamp, + "sha256_root_hash": base64.b64encode(root_hash), + "tree_head_signature": base64.b64encode(tree_head_signature)} + +check_sth_signature(ctbaseurl, sth) + +print "root hash", base64.b16encode(root_hash) + for frontendnode in frontendnodes: print "distributing for node", frontendnode curpos = get_curpos(frontendnode) @@ -154,4 +179,4 @@ for frontendnode in frontendnodes: for missingentry in missingentries: hash = base64.b64decode(missingentry) sendentry(frontendnode, read_chain(hash), hash) - sendsth(frontendnode, {"tree_size": len(logorder)}) + sendsth(frontendnode, sth) -- cgit v1.1