Cleanup tests and use urllib2.build_openercleanup-tests

Remove unused files
Generate test config files directly in release directory
Move test database files to "tests" directory
Generate log key when preparing tests
Report error when STH not found in v1.erl
Make merge, fetchallcerts, submitcert, verifysct, and testcase1 take log key as argument

19 files changed, 134 insertions, 183 deletions

diff --git a/.gitignore b/.gitignore
index ca93487..be00b1a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 *.beam
 rel
 test/test.mk
+*.pyc
diff --git a/Makefile b/Makefile
index 58a1160..261def1 100644
--- a/Makefile
+++ b/Makefile
@@ -1,48 +1,44 @@
+RELDIR=rel
+
 build all:
 	./make.erl
 clean:
 	-rm ebin/*.beam
 release: all
-	rm -rf rel
-	mkdir rel
+	rm -rf $(RELDIR)
+	mkdir $(RELDIR)
 	./makerelease.erl
-	(cd rel; ln -s ../../plop/test .)
-	cp httpd_props.conf rel
-	cp catlfish.config rel
-	cp storage_node.config rel
-	cp storage_node_httpd.conf rel
-	mkdir rel/catlfish
-	cp -r webroot rel/catlfish
-	test -d rel/catlfish/webroot/log || mkdir rel/catlfish/webroot/log
+	mkdir $(RELDIR)/catlfish
+	cp -r webroot $(RELDIR)/catlfish
 
 -include test/test.mk
 
 tests-prepare:
-	rm -r rel/mergedb || true
-	mkdir rel/mergedb
-	mkdir rel/mergedb/chains
-	touch rel/mergedb/logorder
-	rm -r rel/known_roots || true
-	mkdir rel/known_roots
-	cp tools/testcerts/roots/* rel/known_roots
-	cp -r test/config/privatekeys rel
-	cp -r test/config/publickeys rel
-	rm -r rel/tests || true
+	rm -r $(RELDIR)/tests || true
+	mkdir $(RELDIR)/tests
+	mkdir $(RELDIR)/tests/keys
+	(cd $(RELDIR)/tests/keys ; ../../../tools/create-key.sh logkey)
+	mkdir $(RELDIR)/tests/mergedb
+	mkdir $(RELDIR)/tests/mergedb/chains
+	touch $(RELDIR)/tests/mergedb/logorder
+	mkdir $(RELDIR)/tests/known_roots
+	cp tools/testcerts/roots/* $(RELDIR)/tests/known_roots
+	cp -r test/config/privatekeys $(RELDIR)/tests
+	cp -r test/config/publickeys $(RELDIR)/tests
 	@for machine in $(MACHINES); do \
-	  tools/compileconfig.py --config=test/catlfish-test.cfg --localconfig test/catlfish-test-local-$$machine.cfg ; \
-	  mkdir -p rel/tests/machine/machine-$$machine/db ; \
-	  touch rel/tests/machine/machine-$$machine/db/index ; \
-	  touch rel/tests/machine/machine-$$machine/db/newentries ; \
+	  (cd $(RELDIR); ../tools/compileconfig.py --config=../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-$$machine.cfg) ; \
+	  mkdir -p $(RELDIR)/tests/machine/machine-$$machine/db ; \
+	  touch $(RELDIR)/tests/machine/machine-$$machine/db/index ; \
+	  touch $(RELDIR)/tests/machine/machine-$$machine/db/newentries ; \
 	done
-	tools/compileconfig.py --config=test/catlfish-test.cfg --localconfig test/catlfish-test-local-signing.cfg
+	(cd $(RELDIR); ../tools/compileconfig.py --config=../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-signing.cfg)
 	@for node in $(NODES); do \
 	  mkdir -p test/nodes/$$node/log ; \
-	  cp test/config/$$node.config rel ; \
 	done
 
 tests-start:
 	@for node in $(NODES); do \
-	  (cd rel ; bin/run_erl -daemon ../test/nodes/$$node/ ../test/nodes/$$node/log/ "exec bin/erl -config $$node") \
+	  (cd $(RELDIR) ; bin/run_erl -daemon ../test/nodes/$$node/ ../test/nodes/$$node/log/ "exec bin/erl -config $$node") \
 	done
 	@for i in 1 2 3 4 5 6 7 8 9 10; do \
 	  echo "waiting for system to start" ; \
@@ -50,7 +46,7 @@ tests-start:
 	  allstarted=1 ; \
 	  notstarted= ; \
 	  for testurl in $(TESTURLS); do \
-	    if curl -s -k https://$$testurl > /dev/null ; then : ; else allstarted=0 ; notstarted="$$testurl $$notstarted" ; fi ; \
+	    if curl -s -k -4 https://$$testurl > /dev/null ; then : ; else allstarted=0 ; notstarted="$$testurl $$notstarted" ; fi ; \
 	    : ; \
 	  done ; \
 	  if [ $$allstarted -eq 1 ]; then break ; \
@@ -58,20 +54,20 @@ tests-start:
 	done
 
 tests-run:
-	@(cd rel && python ../tools/testcase1.py ) || (echo "Tests failed" ; false)
-	@(cd rel && python ../tools/fetchallcerts.py $(BASEURL)) || (echo "Verification failed" ; false)
-	@(cd rel && rm -f submittedcerts)
-	@(cd rel && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert1.txt --check-sct --sct-file=submittedcerts $(BASEURL)) || (echo "Submission failed" ; false)
-	@(cd rel && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert2.txt --check-sct --sct-file=submittedcerts $(BASEURL)) || (echo "Submission failed" ; false)
-	@(cd rel && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert3.txt --check-sct --sct-file=submittedcerts $(BASEURL)) || (echo "Submission failed" ; false)
-	@(cd rel && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert4.txt --check-sct --sct-file=submittedcerts $(BASEURL)) || (echo "Submission failed" ; false)
-	@(cd rel && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert5.txt --check-sct --sct-file=submittedcerts $(BASEURL)) || (echo "Submission failed" ; false)
-	@(cd rel && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre1.txt --check-sct --sct-file=submittedcerts $(BASEURL)) || (echo "Submission failed" ; false)
-	@(cd rel && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre2.txt --check-sct --sct-file=submittedcerts $(BASEURL)) || (echo "Submission failed" ; false)
-	@(cd rel && python ../tools/merge.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
+	@(cd $(RELDIR) && python ../tools/testcase1.py https://localhost:8080/ tests/keys/logkey.pem) || (echo "Tests failed" ; false)
+	@(cd $(RELDIR) && python ../tools/fetchallcerts.py $(BASEURL) --publickey=tests/keys/logkey.pem) || (echo "Verification failed" ; false)
+	@(cd $(RELDIR) && rm -f submittedcerts)
+	@(cd $(RELDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert1.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem) || (echo "Submission failed" ; false)
+	@(cd $(RELDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert2.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem) || (echo "Submission failed" ; false)
+	@(cd $(RELDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert3.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem) || (echo "Submission failed" ; false)
+	@(cd $(RELDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert4.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem) || (echo "Submission failed" ; false)
+	@(cd $(RELDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert5.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem) || (echo "Submission failed" ; false)
+	@(cd $(RELDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre1.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem) || (echo "Submission failed" ; false)
+	@(cd $(RELDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre2.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem) || (echo "Submission failed" ; false)
+	@(cd $(RELDIR) && python ../tools/merge.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false)
 
 tests-run2:
-	@(cd rel ; python ../tools/verifysct.py --sct-file=submittedcerts --parallel 1 $(BASEURL)) || echo "Verification of SCT:s failed"
+	@(cd $(RELDIR) ; python ../tools/verifysct.py --sct-file=submittedcerts --parallel 1 $(BASEURL) --publickey=tests/keys/logkey.pem) || echo "Verification of SCT:s failed"
 
 tests-stop:
 	@for node in $(NODES); do \
diff --git a/catlfish.config b/catlfish.config
deleted file mode 100644
index 91868e5..0000000
--- a/catlfish.config
+++ /dev/null
@@ -1,27 +0,0 @@
-%% catlfish configuration file (-*- erlang -*-)
-%% Start like this:
-%% $ erl -boot start_sasl -config catlfish -run inets
-[{sasl,
-  [{sasl_error_logger, false},
-   {errlog_type, error},
-   {error_logger_mf_dir, "log"},
-   {error_logger_mf_maxbytes, 10485760},	% 10 MB
-   {error_logger_mf_maxfiles, 10}]},
- {catlfish,
-  [{known_roots_path, "known_roots"},
-   {https_servers,
-    [{external_https_api, "127.0.0.1", 8080, v1}
-    ]},
-   {https_certfile, "catlfish/webroot/certs/webcert.pem"},
-   {https_keyfile, "catlfish/webroot/keys/webkey.pem"},
-   {https_cacertfile, "catlfish/webroot/certs/webcert.pem"}
-  ]},
- {plop,
-  [{entry_root_path, "db/certentries/"},
-   {index_path, "db/index"},
-   {entryhash_root_path, "db/entryhash/"},
-   {treesize_path, "db/treesize"},
-   {indexforhash_root_path, "db/certindex/"},
-   %{storage_nodes, ["https://127.0.0.1:8081/ct/storage/"]},
-   {storage_nodes_quorum, 1}
-  ]}].
diff --git a/httpd_props.conf b/httpd_props.conf
deleted file mode 100644
index 9ea7b30..0000000
--- a/httpd_props.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-%%% Copyright (c) 2014, NORDUnet A/S.
-%%% See LICENSE for licensing information.
-[
- {port, 8080},
- {bind_address, {127, 0, 0, 1}},
- {server_name, "flimsy"},
- {server_root, "catlfish/webroot"},
- {document_root, "catlfish/webroot/docroot"},
- {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head,
-            mod_log, mod_disk_log]},
- {erl_script_alias, {"/ct", [v1, frontend]}},
- {erl_script_nocache, true},
- {error_log, "log/error"},
- {security_log, "log/security"},
- {transfer_log, "log/transfer"},
- {socket_type,
-  {essl,                              % See ssl(3erl) for SSL options.
-   [{versions, ['tlsv1.2', 'tlsv1.1', 'tlsv1']},
-    {certfile, "catlfish/webroot/certs/webcert.pem"},
-    {keyfile, "catlfish/webroot/keys/webkey.pem"},
-    {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}}
-].
diff --git a/src/v1.erl b/src/v1.erl
index e672182..e2cadb3 100644
--- a/src/v1.erl
+++ b/src/v1.erl
@@ -15,8 +15,13 @@ request(post, "ct/v1/add-pre-chain", Input) ->
     add_chain(Input, precert);
 
 request(get, "ct/v1/get-sth", _Query) ->
-    R = plop:sth(),
-    success(R);
+    case plop:sth() of
+        noentry ->
+            lager:error("No valid STH found"),
+            internalerror("No valid STH found");
+        R ->
+            success(R)
+    end;
 
 request(get, "ct/v1/get-sth-consistency", Query) ->
     case lists:sort(Query) of
@@ -109,6 +114,13 @@ html(Text, Input) ->
 success(Data) ->
     {200, [{"Content-Type", "text/json"}], mochijson2:encode(Data)}.
 
+internalerror(Text) ->
+    {500, [{"Content-Type", "text/html"}],
+     io_lib:format(
+       "<html><body><p>~n" ++
+           "~s~n" ++
+           "</body></html>~n", [Text])}.
+
 -spec add_chain(any(), normal|precert) -> any().
 add_chain(Input, Type) ->
     case (catch mochijson2:decode(Input)) of
diff --git a/storage_node.config b/storage_node.config
deleted file mode 100644
index 47a1326..0000000
--- a/storage_node.config
+++ /dev/null
@@ -1,19 +0,0 @@
-%% catlfish configuration file (-*- erlang -*-)
-%% Start like this:
-%% $ erl -boot start_sasl -config catlfish -run inets
-[{sasl,
-  [{sasl_error_logger, false},
-   {errlog_type, error},
-   {error_logger_mf_dir, "log"},
-   {error_logger_mf_maxbytes, 10485760},	% 10 MB
-   {error_logger_mf_maxfiles, 10}]},
- {inets,
-  [{services,
-    [{httpd, [{proplist_file, "storage_node_httpd.conf"}]}]}]},
- {plop,
-  [{entry_root_path, "db/certentries/"},
-   {index_path, "db/index"},
-   {newentries_path, "db/newentries"},
-   {entryhash_root_path, "db/entryhash/"},
-   {treesize_path, "db/treesize"},
-   {indexforhash_root_path, "db/certindex/"}]}].
diff --git a/storage_node_httpd.conf b/storage_node_httpd.conf
deleted file mode 100644
index 2f271f8..0000000
--- a/storage_node_httpd.conf
+++ /dev/null
@@ -1,21 +0,0 @@
-%%% Copyright (c) 2014, NORDUnet A/S.
-%%% See LICENSE for licensing information.
-[
- {port, 8081},
- {bind_address, {127, 0, 0, 1}},
- {server_name, "flimsy"},
- {server_root, "catlfish/webroot"},
- {document_root, "catlfish/webroot/docroot"},
- {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head,
-            mod_log, mod_disk_log]},
- {erl_script_alias, {"/ct", [storage]}},
- {erl_script_nocache, true},
- {error_log, "log/error_storage"},
- {security_log, "log/security_storage"},
- {transfer_log, "log/transfer_storage"},
- {socket_type,
-  {essl,                              % See ssl(3erl) for SSL options.
-   [{certfile, "catlfish/webroot/certs/webcert.pem"},
-    {keyfile, "catlfish/webroot/keys/webkey.pem"},
-    {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}}
-].
diff --git a/test/catlfish-test-local-1.cfg b/test/catlfish-test-local-1.cfg
index 608d4c0..7caacdc 100644
--- a/test/catlfish-test-local-1.cfg
+++ b/test/catlfish-test-local-1.cfg
@@ -10,15 +10,15 @@ publicaddresses:
         frontend-1: 127.0.0.1:8080
 
 paths:
-        configdir: test/config/
-        knownroots: known_roots
+        configdir: .
+        knownroots: tests/known_roots
         https_certfile: catlfish/webroot/certs/webcert.pem
         https_keyfile: catlfish/webroot/keys/webkey.pem
         https_cacertfile: catlfish/webroot/certs/webcert.pem
         db: tests/machine/machine-1/db/
-        publickeys: publickeys
-        logpublickey: test/eckey-public.pem
-        privatekeys: privatekeys
+        publickeys: tests/publickeys
+        logpublickey: tests/keys/logkey.pem
+        privatekeys: tests/privatekeys
 
 #options:
 #        - sctcaching
diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg
index b7f5009..4a77708 100644
--- a/test/catlfish-test-local-merge.cfg
+++ b/test/catlfish-test-local-merge.cfg
@@ -1,8 +1,8 @@
 nodename: merge-1
 
 paths:
-        mergedb: ../rel/mergedb
+        mergedb: tests/mergedb
         https_cacertfile: catlfish/webroot/certs/webcert.pem
-        publickeys: publickeys
-        logpublickey: test/eckey-public.pem
-        privatekeys: privatekeys
+        publickeys: tests/publickeys
+        logpublickey: tests/keys/logkey.pem
+        privatekeys: tests/privatekeys
diff --git a/test/catlfish-test-local-signing.cfg b/test/catlfish-test-local-signing.cfg
index 2cc4df2..b08bf2f 100644
--- a/test/catlfish-test-local-signing.cfg
+++ b/test/catlfish-test-local-signing.cfg
@@ -5,11 +5,11 @@ addresses:
         signing-1: 127.0.0.1:8088
 
 paths:
-        configdir: test/config/
+        configdir: .
         https_certfile: catlfish/webroot/certs/webcert.pem
         https_keyfile: catlfish/webroot/keys/webkey.pem
         https_cacertfile: catlfish/webroot/certs/webcert.pem
-        publickeys: publickeys
-        logpublickey: test/eckey-public.pem
-        logprivatekey: test/eckey.pem
-        privatekeys: privatekeys
+        publickeys: tests/publickeys
+        logpublickey: tests/keys/logkey.pem
+        logprivatekey: tests/keys/logkey-private.pem
+        privatekeys: tests/privatekeys
diff --git a/tools/certkeys.py b/tools/certkeys.py
index 52d61be..43646ef 100644
--- a/tools/certkeys.py
+++ b/tools/certkeys.py
@@ -4,14 +4,6 @@ publickeys = {
     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTD"
     "M0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==",
 
-    "https://127.0.0.1:8080/":
-    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9"
-    "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==",
-
-    "https://localhost:8080/":
-    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9"
-    "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==",
-
     "https://flimsy.ct.nordu.net/":
     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9"
     "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==",
diff --git a/tools/certtools.py b/tools/certtools.py
index 2c97dfb..da5021a 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -88,8 +88,15 @@ def get_root_cert(issuer):
 
     return root_cert
 
+def urlopen(url, data=None):
+    try:
+        opener = urllib2.build_opener(urllib2.HTTPSHandler(context=None))
+    except TypeError:
+        opener = urllib2.build_opener(urllib2.HTTPSHandler())
+    return opener.open(url, data)
+
 def get_sth(baseurl):
-    result = urllib2.urlopen(baseurl + "ct/v1/get-sth").read()
+    result = urlopen(baseurl + "ct/v1/get-sth").read()
     return json.loads(result)
 
 def get_proof_by_hash(baseurl, hash, tree_size):
@@ -97,7 +104,7 @@ def get_proof_by_hash(baseurl, hash, tree_size):
         params = urllib.urlencode({"hash":base64.b64encode(hash),
                                    "tree_size":tree_size})
         result = \
-          urllib2.urlopen(baseurl + "ct/v1/get-proof-by-hash?" + params).read()
+          urlopen(baseurl + "ct/v1/get-proof-by-hash?" + params).read()
         return json.loads(result)
     except urllib2.HTTPError, e:
         print "ERROR:", e.read()
@@ -108,7 +115,7 @@ def get_consistency_proof(baseurl, tree_size1, tree_size2):
         params = urllib.urlencode({"first":tree_size1,
                                    "second":tree_size2})
         result = \
-          urllib2.urlopen(baseurl + "ct/v1/get-sth-consistency?" + params).read()
+          urlopen(baseurl + "ct/v1/get-sth-consistency?" + params).read()
         return json.loads(result)["consistency"]
     except urllib2.HTTPError, e:
         print "ERROR:", e.read()
@@ -131,7 +138,7 @@ def unpack_tls_array(packed_data, length_len):
 
 def add_chain(baseurl, submission):
     try:
-        result = urllib2.urlopen(baseurl + "ct/v1/add-chain", json.dumps(submission)).read()
+        result = urlopen(baseurl + "ct/v1/add-chain", json.dumps(submission)).read()
         return json.loads(result)
     except urllib2.HTTPError, e:
         print "ERROR", e.code,":", e.read()
@@ -148,7 +155,7 @@ def add_chain(baseurl, submission):
 
 def add_prechain(baseurl, submission):
     try:
-        result = urllib2.urlopen(baseurl + "ct/v1/add-pre-chain",
+        result = urlopen(baseurl + "ct/v1/add-pre-chain",
             json.dumps(submission)).read()
         return json.loads(result)
     except urllib2.HTTPError, e:
@@ -167,7 +174,7 @@ def add_prechain(baseurl, submission):
 def get_entries(baseurl, start, end):
     try:
         params = urllib.urlencode({"start":start, "end":end})
-        result = urllib2.urlopen(baseurl + "ct/v1/get-entries?" + params).read()
+        result = urlopen(baseurl + "ct/v1/get-entries?" + params).read()
         return json.loads(result)
     except urllib2.HTTPError, e:
         print "ERROR:", e.read()
@@ -198,8 +205,9 @@ def encode_signature(hash_alg, signature_alg, unpacked_signature):
     signature += tls_array(unpacked_signature, 2)
     return signature
 
-def check_signature(baseurl, signature, data):
-    publickey = base64.decodestring(publickeys[baseurl])
+def check_signature(baseurl, signature, data, publickey=None):
+    if publickey == None:
+        publickey = base64.decodestring(publickeys[baseurl])
     (hash_alg, signature_alg, unpacked_signature) = decode_signature(signature)
     assert hash_alg == 4, \
         "hash_alg is %d, expected 4" % (hash_alg,) # sha256
@@ -230,20 +238,25 @@ def check_auth_header(authheader, expected_key, publickeydir, data, path):
     return True
 
 def http_request(url, data=None, key=None, verifynode=None, publickeydir="."):
-    req = urllib2.Request(url, data)
+    try:
+        opener = urllib2.build_opener(urllib2.HTTPSHandler(context=None))
+    except TypeError:
+        opener = urllib2.build_opener(urllib2.HTTPSHandler())
+
     (keyname, keyfile) = key
     privatekey = get_eckey_from_file(keyfile)
     sk = ecdsa.SigningKey.from_der(privatekey)
     parsed_url = urlparse.urlparse(url)
     if data == None:
-        data = parsed_url.query
+        data_to_sign = parsed_url.query
         method = "GET"
     else:
+        data_to_sign = data
         method = "POST"
-    signature = sk.sign("%s\0%s\0%s" % (method, parsed_url.path, data), hashfunc=hashlib.sha256,
+    signature = sk.sign("%s\0%s\0%s" % (method, parsed_url.path, data_to_sign), hashfunc=hashlib.sha256,
                         sigencode=ecdsa.util.sigencode_der)
-    req.add_header('X-Catlfish-Auth', base64.b64encode(signature) + ";key=" + keyname)
-    result = urllib2.urlopen(req)
+    opener.addheaders = [('X-Catlfish-Auth', base64.b64encode(signature) + ";key=" + keyname)]
+    result = opener.open(url, data)
     authheader = result.info().get('X-Catlfish-Auth')
     data = result.read()
     check_auth_header(authheader, verifynode, publickeydir, data, parsed_url.path)
@@ -263,7 +276,7 @@ def create_signature(baseurl, data, key=None):
     unpacked_signature = get_signature(baseurl, data, key)
     return encode_signature(4, 3, unpacked_signature)
 
-def check_sth_signature(baseurl, sth):
+def check_sth_signature(baseurl, sth, publickey=None):
     signature = base64.decodestring(sth["tree_head_signature"])
 
     version = struct.pack(">b", 0)
@@ -273,7 +286,7 @@ def check_sth_signature(baseurl, sth):
     hash = base64.decodestring(sth["sha256_root_hash"])
     tree_head = version + signature_type + timestamp + tree_size + hash
 
-    check_signature(baseurl, signature, tree_head)
+    check_signature(baseurl, signature, tree_head, publickey=publickey)
 
 def create_sth_signature(tree_size, timestamp, root_hash, baseurl, key=None):
     version = struct.pack(">b", 0)
@@ -284,8 +297,9 @@ def create_sth_signature(tree_size, timestamp, root_hash, baseurl, key=None):
 
     return create_signature(baseurl, tree_head, key=key)
 
-def check_sct_signature(baseurl, signed_entry, sct, precert=False):
-    publickey = base64.decodestring(publickeys[baseurl])
+def check_sct_signature(baseurl, signed_entry, sct, precert=False, publickey=None):
+    if publickey == None:
+        publickey = base64.decodestring(publickeys[baseurl])
     calculated_logid = hashlib.sha256(publickey).digest()
     received_logid = base64.decodestring(sct["id"])
     assert calculated_logid == received_logid, \
@@ -306,7 +320,7 @@ def check_sct_signature(baseurl, signed_entry, sct, precert=False):
       entry_type + signed_entry + \
       tls_array(base64.decodestring(sct["extensions"]), 2)
 
-    check_signature(baseurl, signature, signed_struct)
+    check_signature(baseurl, signature, signed_struct, publickey=publickey)
 
 def pack_mtl(timestamp, leafcert):
     entry_type = struct.pack(">H", 0)
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 4996994..c239bd0 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -158,7 +158,7 @@ def gen_config(nodename, config, localconfig):
     bind_publichttpaddress = localconfig.get("publichttpaddresses", {}).get(nodename)
     options = localconfig.get("options", [])
 
-    configfile = open(paths["configdir"] + nodename + ".config", "w")
+    configfile = open(paths["configdir"] + "/" + nodename + ".config", "w")
     print >>configfile, "%% catlfish configuration file (-*- erlang -*-)"
 
     (nodetype, nodeconfig) = get_node_config(nodename, config)
diff --git a/tools/create-key.sh b/tools/create-key.sh
new file mode 100755
index 0000000..9d29c86
--- /dev/null
+++ b/tools/create-key.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+openssl ecparam -name prime256v1 -genkey -noout -out $1-private.pem
+openssl ec -in $1-private.pem -pubout -out $1.pem
diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py
index e0ea92f..395fe69 100755
--- a/tools/fetchallcerts.py
+++ b/tools/fetchallcerts.py
@@ -22,6 +22,7 @@ parser = argparse.ArgumentParser(description='')
 parser.add_argument('baseurl', help="Base URL for CT server")
 parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir')
 parser.add_argument('--write-sth', action='store_true', help='Write STH')
+parser.add_argument('--publickey', default=None, metavar="file", help='Public key for the CT log')
 args = parser.parse_args()
 
 def get_entries_wrapper(baseurl, start, end):
@@ -39,8 +40,10 @@ def print_layer(layer):
     for entry in layer:
         print base64.b16encode(entry)
 
+logpublickey = get_public_key_from_file(args.publickey) if args.publickey else None
+
 sth = get_sth(args.baseurl)
-check_sth_signature(args.baseurl, sth)
+check_sth_signature(args.baseurl, sth, publickey=logpublickey)
 tree_size = sth["tree_size"]
 root_hash = base64.decodestring(sth["sha256_root_hash"])
 
diff --git a/tools/merge.py b/tools/merge.py
index e6fae24..f9c93d9 100755
--- a/tools/merge.py
+++ b/tools/merge.py
@@ -16,7 +16,9 @@ import hashlib
 import urlparse
 import os
 import yaml
-from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point, http_request
+from certtools import build_merkle_tree, create_sth_signature, \
+    check_sth_signature, get_eckey_from_file, timing_point, http_request, \
+    get_public_key_from_file
 
 parser = argparse.ArgumentParser(description="")
 parser.add_argument('--config', help="System configuration", required=True)
@@ -41,6 +43,8 @@ logorderfile = mergedb + "/logorder"
 
 own_key = (localconfig["nodename"], "%s/%s-private.pem" % (paths["privatekeys"], localconfig["nodename"]))
 
+logpublickey = get_public_key_from_file(paths["logpublickey"])
+
 hashed_dir = True
 
 def parselogrow(row):
@@ -238,19 +242,23 @@ tree_size = len(logorder)
 root_hash = tree[-1][0]
 timestamp = int(time.time() * 1000)
 
+tree_head_signature = None
 for signingnode in signingnodes:
     try:
         tree_head_signature = create_sth_signature(tree_size, timestamp,
                                                    root_hash, "https://%s/" % signingnode["address"], key=own_key)
         break
-    except urllib2.URLError:
-        pass
+    except urllib2.URLError, e:
+        print e
+if tree_head_signature == None:
+    print >>sys.stderr, "Could not contact any signing nodes"
+    sys.exit(1)
 
 sth = {"tree_size": tree_size, "timestamp": timestamp,
        "sha256_root_hash": base64.b64encode(root_hash),
        "tree_head_signature": base64.b64encode(tree_head_signature)}
 
-check_sth_signature(ctbaseurl, sth)
+check_sth_signature(ctbaseurl, sth, publickey=logpublickey)
 
 timing_point(timing, "build sth")
 
diff --git a/tools/submitcert.py b/tools/submitcert.py
index 2e8cc33..ba4b337 100755
--- a/tools/submitcert.py
+++ b/tools/submitcert.py
@@ -30,6 +30,7 @@ parser.add_argument('--sct-file', default=None, metavar="file", help='Store SCT:
 parser.add_argument('--parallel', type=int, default=16, metavar="n", help="Number of parallel submits")
 parser.add_argument('--check-sct', action='store_true', help="Check SCT signature")
 parser.add_argument('--pre-warm', action='store_true', help="Wait 3 seconds after first submit")
+parser.add_argument('--publickey', default=None, metavar="file", help='Public key for the CT log')
 args = parser.parse_args()
 
 from multiprocessing import Pool
@@ -37,6 +38,8 @@ from multiprocessing import Pool
 baseurl = args.baseurl
 certfilepath = args.store
 
+logpublickey = get_public_key_from_file(args.publickey) if args.publickey else None
+
 lookup_in_log = False
 
 if certfilepath[-1] == "/":
@@ -84,7 +87,7 @@ def submitcert((certfile, cert)):
 
     try:
         if args.check_sct:
-            check_sct_signature(baseurl, signed_entry, result, precert=precert)
+            check_sct_signature(baseurl, signed_entry, result, precert=precert, publickey=logpublickey)
             timing_point(timing, "checksig")
     except AssertionError, e:
         print "ERROR:", certfile, e
diff --git a/tools/testcase1.py b/tools/testcase1.py
index 4502b56..1d46230 100755
--- a/tools/testcase1.py
+++ b/tools/testcase1.py
@@ -14,7 +14,9 @@ import hashlib
 import itertools
 from certtools import *
 
-baseurls = ["https://127.0.0.1:8080/"]
+baseurls = [sys.argv[1]]
+logpublickeyfile = sys.argv[2]
+
 certfiles = ["../tools/testcerts/cert1.txt", "../tools/testcerts/cert2.txt",
              "../tools/testcerts/cert3.txt", "../tools/testcerts/cert4.txt",
              "../tools/testcerts/cert5.txt"]
@@ -28,6 +30,8 @@ cc5 = get_certs_from_file(certfiles[4])
 failures = 0
 indentation = ""
 
+logpublickey = get_public_key_from_file(logpublickeyfile)
+
 def testgroup(name):
     global indentation
     print name + ":"
@@ -55,7 +59,7 @@ def print_and_check_tree_size(expected, baseurl):
     global failures
     sth = get_sth(baseurl)
     try:
-        check_sth_signature(baseurl, sth)
+        check_sth_signature(baseurl, sth, publickey=logpublickey)
     except AssertionError, e:
         print_error("%s", e)
     except ecdsa.keys.BadSignatureError, e:
@@ -71,13 +75,13 @@ def do_add_chain(chain, baseurl):
         print_error("%s", e)
     try:
         signed_entry = pack_cert(chain[0])
-        check_sct_signature(baseurl, signed_entry, result)
+        check_sct_signature(baseurl, signed_entry, result, publickey=logpublickey)
+        print_success("signature check succeeded")
     except AssertionError, e:
         print_error("%s", e)
     except ecdsa.keys.BadSignatureError, e:
         print e
         print_error("bad SCT signature")
-    print_success("signature check succeeded")
     return result
 
 def get_and_validate_proof(timestamp, chain, leaf_index, nentries, baseurl):
diff --git a/tools/verifysct.py b/tools/verifysct.py
index 27ab4c9..4b8e38a 100755
--- a/tools/verifysct.py
+++ b/tools/verifysct.py
@@ -22,12 +22,15 @@ parser = argparse.ArgumentParser(description='')
 parser.add_argument('baseurl', help="Base URL for CT server")
 parser.add_argument('--sct-file', default=None, metavar="dir", help='SCT:s to verify')
 parser.add_argument('--parallel', type=int, default=16, metavar="n", help="Number of parallel verifications")
+parser.add_argument('--publickey', default=None, metavar="file", help='Public key for the CT log')
 args = parser.parse_args()
 
 from multiprocessing import Pool
 
 baseurl = args.baseurl
 
+logpublickey = get_public_key_from_file(args.publickey) if args.publickey else None
+
 sth = get_sth(baseurl)
 
 def verifysct(sctentry):
@@ -43,7 +46,7 @@ def verifysct(sctentry):
             signed_entry = pack_precert(leafcert, issuer_key_hash)
         else:
             signed_entry = pack_cert(leafcert)
-        check_sct_signature(baseurl, signed_entry, sctentry["sct"], precert=issuer_key_hash)
+        check_sct_signature(baseurl, signed_entry, sctentry["sct"], precert=issuer_key_hash, publickey=logpublickey)
         timing_point(timing, "checksig")
     except AssertionError, e:
         print "ERROR:", e