summaryrefslogtreecommitdiff
path: root/tools/merge.py
diff options
context:
space:
mode:
authorLinus Nordberg <linus@nordberg.se>2015-03-23 16:04:10 +0100
committerLinus Nordberg <linus@nordberg.se>2015-03-23 16:04:10 +0100
commit0fd4141586310d7735a376bdbf4541b0d8e88169 (patch)
tree73886c42f951dbffefd52dea94af3365951c2e2b /tools/merge.py
parent056b69cc4891c5ef1ba8da7e0a9369cb069a03e0 (diff)
parent3f9f38468dd9f43a7a71768dbe84dd40723c30c5 (diff)
Merge branch 'precert2'
Conflicts: Makefile
Diffstat (limited to 'tools/merge.py')
-rwxr-xr-xtools/merge.py76
1 files changed, 43 insertions, 33 deletions
diff --git a/tools/merge.py b/tools/merge.py
index 6becf7e..1b94581 100755
--- a/tools/merge.py
+++ b/tools/merge.py
@@ -14,14 +14,15 @@ import time
import ecdsa
import hashlib
import urlparse
-from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point
+import os
+from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point, http_request
parser = argparse.ArgumentParser(description="")
parser.add_argument("--baseurl", metavar="url", help="Base URL for CT server", required=True)
parser.add_argument("--frontend", action="append", metavar="url", help="Base URL for frontend server", required=True)
parser.add_argument("--storage", action="append", metavar="url", help="Base URL for storage server", required=True)
parser.add_argument("--mergedb", metavar="dir", help="Merge database directory", required=True)
-parser.add_argument("--keyfile", metavar="keyfile", help="File containing log key", required=True)
+parser.add_argument("--signing", metavar="url", help="Base URL for signing server", required=True)
parser.add_argument("--own-keyname", metavar="keyname", help="The key name of the merge node", required=True)
parser.add_argument("--own-keyfile", metavar="keyfile", help="The file containing the private key of the merge node", required=True)
parser.add_argument("--nomerge", action='store_true', help="Don't actually do merge")
@@ -34,6 +35,10 @@ storagenodes = args.storage
chainsdir = args.mergedb + "/chains"
logorderfile = args.mergedb + "/logorder"
+own_key = (args.own_keyname, args.own_keyfile)
+
+hashed_dir = True
+
def parselogrow(row):
return base64.b16decode(row)
@@ -42,12 +47,26 @@ def get_logorder():
return [parselogrow(row.rstrip()) for row in f]
def write_chain(key, value):
- f = open(chainsdir + "/" + base64.b16encode(key), "w")
+ filename = base64.b16encode(key)
+ if hashed_dir:
+ path = chainsdir + "/" + filename[0:2] + "/" + filename[2:4] + "/" + filename[4:6]
+ try:
+ os.makedirs(path)
+ except Exception, e:
+ print e
+ else:
+ path = chainsdir
+ f = open(path + "/" + filename, "w")
f.write(value)
f.close()
def read_chain(key):
- f = open(chainsdir + "/" + base64.b16encode(key), "r")
+ filename = base64.b16encode(key)
+ path = chainsdir + "/" + filename[0:2] + "/" + filename[2:4] + "/" + filename[4:6]
+ try:
+ f = open(path + "/" + filename, "r")
+ except IOError, e:
+ f = open(chainsdir + "/" + filename, "r")
value = f.read()
f.close()
return value
@@ -57,26 +76,9 @@ def add_to_logorder(key):
f.write(base64.b16encode(key) + "\n")
f.close()
-def http_request(url, data=None):
- req = urllib2.Request(url, data)
- keyname = args.own_keyname
- privatekey = get_eckey_from_file(args.own_keyfile)
- sk = ecdsa.SigningKey.from_der(privatekey)
- parsed_url = urlparse.urlparse(url)
- if data == None:
- data = parsed_url.query
- method = "GET"
- else:
- method = "POST"
- signature = sk.sign("%s\0%s\0%s" % (method, parsed_url.path, data), hashfunc=hashlib.sha256,
- sigencode=ecdsa.util.sigencode_der)
- req.add_header('X-Catlfish-Auth', base64.b64encode(signature) + ";key=" + keyname)
- result = urllib2.urlopen(req).read()
- return result
-
def get_new_entries(baseurl):
try:
- result = http_request(baseurl + "ct/storage/fetchnewentries")
+ result = http_request(baseurl + "ct/storage/fetchnewentries", key=own_key)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return [base64.b64decode(entry) for entry in parsed_result[u"entries"]]
@@ -89,7 +91,7 @@ def get_new_entries(baseurl):
def get_entries(baseurl, hashes):
try:
params = urllib.urlencode({"hash":[base64.b64encode(hash) for hash in hashes]}, doseq=True)
- result = http_request(baseurl + "ct/storage/getentry?" + params)
+ result = http_request(baseurl + "ct/storage/getentry?" + params, key=own_key)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
entries = dict([(base64.b64decode(entry["hash"]), base64.b64decode(entry["entry"])) for entry in parsed_result[u"entries"]])
@@ -104,7 +106,7 @@ def get_entries(baseurl, hashes):
def get_curpos(baseurl):
try:
- result = http_request(baseurl + "ct/frontend/currentposition")
+ result = http_request(baseurl + "ct/frontend/currentposition", key=own_key)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return parsed_result[u"position"]
@@ -117,7 +119,7 @@ def get_curpos(baseurl):
def sendlog(baseurl, submission):
try:
result = http_request(baseurl + "ct/frontend/sendlog",
- json.dumps(submission))
+ json.dumps(submission), key=own_key)
return json.loads(result)
except urllib2.HTTPError, e:
print "ERROR: sendlog", e.read()
@@ -133,7 +135,7 @@ def sendlog(baseurl, submission):
def sendentry(baseurl, entry, hash):
try:
result = http_request(baseurl + "ct/frontend/sendentry",
- json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)}))
+ json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)}), key=own_key)
return json.loads(result)
except urllib2.HTTPError, e:
print "ERROR: sendentry", e.read()
@@ -149,7 +151,7 @@ def sendentry(baseurl, entry, hash):
def sendsth(baseurl, submission):
try:
result = http_request(baseurl + "ct/frontend/sendsth",
- json.dumps(submission))
+ json.dumps(submission), key=own_key)
return json.loads(result)
except urllib2.HTTPError, e:
print "ERROR: sendsth", e.read()
@@ -164,7 +166,7 @@ def sendsth(baseurl, submission):
def get_missingentries(baseurl):
try:
- result = http_request(baseurl + "ct/frontend/missingentries")
+ result = http_request(baseurl + "ct/frontend/missingentries", key=own_key)
parsed_result = json.loads(result)
if parsed_result.get(u"result") == u"ok":
return parsed_result[u"entries"]
@@ -230,10 +232,9 @@ tree = build_merkle_tree(logorder)
tree_size = len(logorder)
root_hash = tree[-1][0]
timestamp = int(time.time() * 1000)
-privatekey = get_eckey_from_file(args.keyfile)
tree_head_signature = create_sth_signature(tree_size, timestamp,
- root_hash, privatekey)
+ root_hash, args.signing, key=own_key)
sth = {"tree_size": tree_size, "timestamp": timestamp,
"sha256_root_hash": base64.b64encode(root_hash),
@@ -255,7 +256,10 @@ for frontendnode in frontendnodes:
print "current position", curpos
entries = [base64.b64encode(entry) for entry in logorder[curpos:]]
for chunk in chunks(entries, 1000):
- sendlog(frontendnode, {"start": curpos, "hashes": chunk})
+ sendlogresult = sendlog(frontendnode, {"start": curpos, "hashes": chunk})
+ if sendlogresult["result"] != "ok":
+ print "sendlog:", sendlogresult
+ sys.exit(1)
curpos += len(chunk)
print curpos,
sys.stdout.flush()
@@ -266,8 +270,14 @@ for frontendnode in frontendnodes:
print "missing entries:", len(missingentries)
for missingentry in missingentries:
hash = base64.b64decode(missingentry)
- sendentry(frontendnode, read_chain(hash), hash)
+ sendentryresult = sendentry(frontendnode, read_chain(hash), hash)
+ if sendentryresult["result"] != "ok":
+ print "send sth:", sendentryresult
+ sys.exit(1)
timing_point(timing, "send missing")
- sendsth(frontendnode, sth)
+ sendsthresult = sendsth(frontendnode, sth)
+ if sendsthresult["result"] != "ok":
+ print "send sth:", sendsthresult
+ sys.exit(1)
timing_point(timing, "send sth")
print timing["deltatimes"]