summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLinus Nordberg <linus@nordu.net>2015-08-19 16:23:50 +0200
committerLinus Nordberg <linus@nordu.net>2015-08-19 16:23:50 +0200
commit3464a089a9ccbac1a8bc92c8dfba402e7e307733 (patch)
tree071448a641b3fb5fffcd2bcb38c1a3e2b52ad7ae
parent83472c419a90deb33d21d2e69632e421f094318f (diff)
parentfae6b9c78099fa40bcbf9ed451979cdca715fef0 (diff)
Merge remote-tracking branch 'refs/remotes/map/storagegc'
Conflicts: Makefile
-rw-r--r--Makefile3
-rw-r--r--tools/certtools.py9
-rwxr-xr-xtools/compileconfig.py1
-rwxr-xr-xtools/storagegc.py66
4 files changed, 79 insertions, 0 deletions
diff --git a/Makefile b/Makefile
index 4cc0330..a8c81bc 100644
--- a/Makefile
+++ b/Makefile
@@ -80,6 +80,7 @@ tests-run:
@(cd $(INSTDIR) && python ../tools/check-sth.py --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem https://localhost:8080/) || (echo "Check failed" ; false)
@(cd $(INSTDIR) && python ../tools/fetchallcerts.py $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Verification failed" ; false)
@(cd $(INSTDIR) && rm -f submittedcerts)
+ @(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg) || (echo "GC failed" ; false)
@(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert1.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false)
@(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert2.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false)
@(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert3.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false)
@@ -87,6 +88,7 @@ tests-run:
@(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert5.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false)
@(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre1.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false)
@(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre2.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false)
+ @(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg) || (echo "GC failed" ; false)
@(cd $(INSTDIR) && python ../tools/merge.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
@diff -r -x nursery -x verifiedsize catlfish/tests/mergedb catlfish/tests/mergedb-secondary || (echo "Merge databases not matching" ; false)
@(cd $(INSTDIR) && python ../tools/check-sth.py --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem https://localhost:8080/) || (echo "Check failed" ; false)
@@ -99,6 +101,7 @@ tests-run:
@(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/cert4.txt fetchcertstore/00000003) || (echo "Verification failed" ; false)
@(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/cert5.txt fetchcertstore/00000004) || (echo "Verification failed" ; false)
@(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/pre1.txt:../tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006) || (echo "Verification failed" ; false)
+ @(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg) || (echo "GC failed" ; false)
tests-run2:
@(cd $(INSTDIR) ; python ../tools/verifysct.py --sct-file=submittedcerts --parallel 1 $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || echo "Verification of SCT:s failed"
diff --git a/tools/certtools.py b/tools/certtools.py
index ed8ab30..dfd5b24 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -15,6 +15,7 @@ import ecdsa
import datetime
import cStringIO
import zipfile
+import shutil
from certkeys import publickeys
def get_cert_info(s):
@@ -734,3 +735,11 @@ def extract_original_entry(entry):
leaf_cert = precert
certchain = decode_certificate_chain(extra_data)
return ([leaf_cert] + certchain, timestamp, issuer_key_hash)
+
+def mv_file(fromfn, tofn):
+ shutil.move(fromfn, tofn)
+
+def write_file(fn, sth):
+ tempname = fn + ".new"
+ open(tempname, 'w').write(json.dumps(sth))
+ mv_file(tempname, fn)
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 1e1fa41..d493226 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -237,6 +237,7 @@ def gen_config(nodename, config, localconfig):
elif nodetype == "storagenodes":
plopconfig += [
(Symbol("newentries_path"), paths["db"] + "newentries"),
+ (Symbol("lastverifiednewentry_path"), paths["db"] + "lastverifiednewentry"),
]
if nodetype in ("frontendnodes", "storagenodes"):
plopconfig += [
diff --git a/tools/storagegc.py b/tools/storagegc.py
new file mode 100755
index 0000000..c13dcb5
--- /dev/null
+++ b/tools/storagegc.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2014, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+import argparse
+import urllib2
+import urllib
+import json
+import base64
+import sys
+import yaml
+from certtools import *
+
+parser = argparse.ArgumentParser(description='')
+parser.add_argument('--config', help="System configuration", required=True)
+parser.add_argument('--localconfig', help="Local configuration", required=True)
+args = parser.parse_args()
+
+config = yaml.load(open(args.config))
+localconfig = yaml.load(open(args.localconfig))
+
+paths = localconfig["paths"]
+db_path = paths["db"]
+create_ssl_context(cafile=paths["https_cacertfile"])
+
+baseurl = config["baseurl"]
+
+sth = get_sth(baseurl)
+
+def verifyleafhash(leaf_hash):
+ try:
+ proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"])
+ except SystemExit:
+ return False
+
+ leaf_index = proof["leaf_index"]
+ inclusion_proof = [base64.b64decode(e) for e in proof["audit_path"]]
+
+ calc_root_hash = verify_inclusion_proof(inclusion_proof, leaf_index, sth["tree_size"], leaf_hash)
+
+ root_hash = base64.b64decode(sth["sha256_root_hash"])
+ if root_hash != calc_root_hash:
+ print "sth calculation incorrect:"
+ print base64.b16encode(root_hash)
+ print base64.b16encode(calc_root_hash)
+ sys.exit(1)
+
+ return True
+
+starttime = datetime.datetime.now()
+
+lastverified = (-1, None)
+
+try:
+ for i, line in enumerate(open(db_path + "newentries")):
+ leaf_hash = base64.b16decode(line.strip(), casefold=True)
+ result = verifyleafhash(leaf_hash)
+ if not result:
+ break
+ lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()}
+ if lastverified["index"] >= 0:
+ write_file(db_path + "lastverifiednewentry", lastverified)
+ print "lastverified", lastverified
+except KeyboardInterrupt:
+ pass