From 194c9aa3b8c463fa487dc9ef7e172332a8d94d72 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 20 Oct 2014 14:33:41 +0200 Subject: Added external merging support --- Makefile | 5 ++ catlfish.config | 5 +- httpd_props.conf | 2 +- storage_node.config | 19 +++++++ storage_node_httpd.conf | 21 ++++++++ tools/merge.py | 133 ++++++++++++++++++++++++++++++++++++++++++++++++ tools/testcase1.py | 13 +++++ 7 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 storage_node.config create mode 100644 storage_node_httpd.conf create mode 100755 tools/merge.py diff --git a/Makefile b/Makefile index 78f8a6b..aa96290 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,13 @@ release: ln -s ../../plop/test .) cp httpd_props.conf rel cp catlfish.config rel + cp storage_node.config rel + cp storage_node_httpd.conf rel mkdir rel/catlfish mkdir rel/db + mkdir rel/mergedb + mkdir rel/mergedb/chains + touch rel/mergedb/logorder printf "0" > rel/db/treesize cp -r webroot rel/catlfish test -d rel/catlfish/webroot/log || mkdir rel/catlfish/webroot/log diff --git a/catlfish.config b/catlfish.config index 75d00fa..672f997 100644 --- a/catlfish.config +++ b/catlfish.config @@ -21,4 +21,7 @@ {index_path, "db/index"}, {entryhash_root_path, "db/entryhash/"}, {treesize_path, "db/treesize"}, - {indexforhash_root_path, "db/certindex/"}]}]. + {indexforhash_root_path, "db/certindex/"}, + %{storage_nodes, ["https://127.0.0.1:8081/ct/storage/"]}, + {storage_nodes_quorum, 1} + ]}]. diff --git a/httpd_props.conf b/httpd_props.conf index ae4c7be..9ea7b30 100644 --- a/httpd_props.conf +++ b/httpd_props.conf @@ -8,7 +8,7 @@ {document_root, "catlfish/webroot/docroot"}, {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head, mod_log, mod_disk_log]}, - {erl_script_alias, {"/ct", [v1]}}, + {erl_script_alias, {"/ct", [v1, frontend]}}, {erl_script_nocache, true}, {error_log, "log/error"}, {security_log, "log/security"}, diff --git a/storage_node.config b/storage_node.config new file mode 100644 index 0000000..47a1326 --- /dev/null +++ b/storage_node.config @@ -0,0 +1,19 @@ +%% catlfish configuration file (-*- erlang -*-) +%% Start like this: +%% $ erl -boot start_sasl -config catlfish -run inets +[{sasl, + [{sasl_error_logger, false}, + {errlog_type, error}, + {error_logger_mf_dir, "log"}, + {error_logger_mf_maxbytes, 10485760}, % 10 MB + {error_logger_mf_maxfiles, 10}]}, + {inets, + [{services, + [{httpd, [{proplist_file, "storage_node_httpd.conf"}]}]}]}, + {plop, + [{entry_root_path, "db/certentries/"}, + {index_path, "db/index"}, + {newentries_path, "db/newentries"}, + {entryhash_root_path, "db/entryhash/"}, + {treesize_path, "db/treesize"}, + {indexforhash_root_path, "db/certindex/"}]}]. diff --git a/storage_node_httpd.conf b/storage_node_httpd.conf new file mode 100644 index 0000000..2f271f8 --- /dev/null +++ b/storage_node_httpd.conf @@ -0,0 +1,21 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. +[ + {port, 8081}, + {bind_address, {127, 0, 0, 1}}, + {server_name, "flimsy"}, + {server_root, "catlfish/webroot"}, + {document_root, "catlfish/webroot/docroot"}, + {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head, + mod_log, mod_disk_log]}, + {erl_script_alias, {"/ct", [storage]}}, + {erl_script_nocache, true}, + {error_log, "log/error_storage"}, + {security_log, "log/security_storage"}, + {transfer_log, "log/transfer_storage"}, + {socket_type, + {essl, % See ssl(3erl) for SSL options. + [{certfile, "catlfish/webroot/certs/webcert.pem"}, + {keyfile, "catlfish/webroot/keys/webkey.pem"}, + {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}} +]. diff --git a/tools/merge.py b/tools/merge.py new file mode 100755 index 0000000..7120d04 --- /dev/null +++ b/tools/merge.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Kungliga Tekniska Högskolan +# (KTH Royal Institute of Technology, Stockholm, Sweden). +# See LICENSE for licensing information. + +import json +import base64 +import urllib +import urllib2 +import sys + +frontendnodes = ["https://127.0.0.1:8080/"] +storagenodes = ["https://127.0.0.1:8081/"] + +chainsdir = "../rel/mergedb/chains" +logorderfile = "../rel/mergedb/logorder" + +def parselogrow(row): + return base64.b16decode(row) + +def get_logorder(): + f = open(logorderfile, "r") + return [parselogrow(row.rstrip()) for row in f] + +def write_chain(key, value): + f = open(chainsdir + "/" + base64.b16encode(key), "w") + f.write(value) + f.close() + +def read_chain(key): + f = open(chainsdir + "/" + base64.b16encode(key), "r") + value = f.read() + f.close() + return value + +def add_to_logorder(key): + f = open(logorderfile, "a") + f.write(base64.b16encode(key) + "\n") + f.close() + +def get_new_entries(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/storage/fetchnewentries").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"entries"] + print "ERROR: fetchnewentries", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: fetchnewentries", e.read() + sys.exit(1) + +def get_curpos(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/currentposition").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"position"] + print "ERROR: currentposition", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: currentposition", e.read() + sys.exit(1) + +def sendlog(baseurl, submission): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendlog", + json.dumps(submission)).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendlog", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e + +def sendsth(baseurl, submission): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendsth", + json.dumps(submission)).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendsth", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e + +def get_missingentries(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/missingentries").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"entries"] + print "ERROR: missingentries", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: missingentries", e.read() + sys.exit(1) + + +logorder = get_logorder() +certsinlog = set(logorder) + +new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] + +for new_entry in new_entries: + hash = base64.b64decode(new_entry["hash"]) + entry = base64.b64decode(new_entry["entry"]) + if hash not in certsinlog: + write_chain(hash, entry) + add_to_logorder(hash) + logorder.append(hash) + certsinlog.add(hash) + print "added", base64.b16encode(hash) + +for frontendnode in frontendnodes: + curpos = get_curpos(frontendnode) + entries = [base64.b64encode(entry) for entry in logorder[curpos:]] + sendlog(frontendnode, {"start": curpos, "hashes": entries}) + missingentries = get_missingentries(frontendnode) + print "missing entries:", missingentries + # XXX: no test case for missing entries yet, waiting to implement + sendsth(frontendnode, {"tree_size": len(logorder)}) diff --git a/tools/testcase1.py b/tools/testcase1.py index eab6c6f..2d5e0e8 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Copyright (c) 2014, NORDUnet A/S. # See LICENSE for licensing information. @@ -125,12 +126,16 @@ testgroup("cert1") result1 = do_add_chain(cc1) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(1) result2 = do_add_chain(cc1) assert_equal(result2["timestamp"], result1["timestamp"], "timestamp") +subprocess.call(["./merge.py"]) + print_and_check_tree_size(1) # TODO: add invalid cert and check that it generates an error @@ -142,6 +147,8 @@ testgroup("cert2") result3 = do_add_chain(cc2) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(2) get_and_validate_proof(result1["timestamp"], cc1, 0, 1) @@ -151,6 +158,8 @@ testgroup("cert3") result4 = do_add_chain(cc3) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(3) get_and_validate_proof(result1["timestamp"], cc1, 0, 2) @@ -161,6 +170,8 @@ testgroup("cert4") result5 = do_add_chain(cc4) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(4) get_and_validate_proof(result1["timestamp"], cc1, 0, 2) @@ -172,6 +183,8 @@ testgroup("cert5") result6 = do_add_chain(cc5) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(5) get_and_validate_proof(result1["timestamp"], cc1, 0, 3) -- cgit v1.1 From bda18b837a255aeb20366c736272698fa3223c4f Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 24 Oct 2014 15:23:33 +0200 Subject: Repair tests to work with x509 validation code. Add intermediate certificates to test chains. --- tools/testcase1.py | 2 +- tools/testcerts/cert3.txt | 30 ++++++++++++++++++++++++++++++ tools/testcerts/cert4.txt | 31 +++++++++++++++++++++++++++++++ tools/testcerts/cert5.txt | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) diff --git a/tools/testcase1.py b/tools/testcase1.py index 2d5e0e8..dd3597e 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -105,7 +105,7 @@ def get_and_check_entry(timestamp, chain, leaf_index): assert_equal(fetchedcert, submittedcert, "cert %d in chain" % (i,)) if len(certchain) == len(submittedcertchain) + 1: - last_issuer = get_cert_info(certs[-1])["issuer"] + last_issuer = get_cert_info(submittedcertchain[-1])["issuer"] root_subject = get_cert_info(certchain[-1])["subject"] if last_issuer == root_subject: print_success("fetched chain has an appended root cert") diff --git a/tools/testcerts/cert3.txt b/tools/testcerts/cert3.txt index d12e485..a776b58 100644 --- a/tools/testcerts/cert3.txt +++ b/tools/testcerts/cert3.txt @@ -38,6 +38,36 @@ SbDmRK4Rxa5UmgfZnezD0snHVUCrzKzP subject=/OU=Domain Control Validated/CN=*.nordu.net issuer=/C=NL/O=TERENA/CN=TERENA SSL CA --- + +Manually added intermediate certificate: +-----BEGIN CERTIFICATE----- +MIIEmDCCA4CgAwIBAgIQS8gUAy8H+mqk8Nop32F5ujANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNMDkwNTE4MDAwMDAwWhcNMjAwNTMwMTA0ODM4WjA2MQswCQYD +VQQGEwJOTDEPMA0GA1UEChMGVEVSRU5BMRYwFAYDVQQDEw1URVJFTkEgU1NMIENB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw+NIxC9cwcupmf0booNd +ij2tOtDipEMfTQ7+NSUwpWkbxOjlwY9UfuFqoppcXN49/ALOlrhfj4NbzGBAkPjk +tjolnF8UUeyx56+eUKExVccCvaxSin81joL6hK0V/qJ/gxA6VVOULAEWdJRUYyij +8lspPZSIgCDiFFkhGbSkmOFg5vLrooCDQ+CtaPN5GYtoQ1E/iptBhQw1jF218bbl +p8ODtWsjb9Sl61DllPFKX+4nSxQSFSRMDc9ijbcAIa06Mg9YC18em9HfnY6pGTVQ +L0GprTvG4EWyUzl/Ib8iGodcNK5Sbwd9ogtOnyt5pn0T3fV/g3wvWl13eHiRoBS/ +fQIDAQABo4IBPjCCATowHwYDVR0jBBgwFoAUoXJfJhsomEOVXQc31YWWnUvSw0Uw +HQYDVR0OBBYEFAy9k2gM896ro0lrKzdXR+qQ47ntMA4GA1UdDwEB/wQEAwIBBjAS +BgNVHRMBAf8ECDAGAQH/AgEAMBgGA1UdIAQRMA8wDQYLKwYBBAGyMQECAh0wRAYD +VR0fBD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VS +Rmlyc3QtSGFyZHdhcmUuY3JsMHQGCCsGAQUFBwEBBGgwZjA9BggrBgEFBQcwAoYx +aHR0cDovL2NydC51c2VydHJ1c3QuY29tL1VUTkFkZFRydXN0U2VydmVyX0NBLmNy +dDAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AudXNlcnRydXN0LmNvbTANBgkqhkiG +9w0BAQUFAAOCAQEATiPuSJz2hYtxxApuc5NywDqOgIrZs8qy1AGcKM/yXA4hRJML +thoh45gBlA5nSYEevj0NTmDa76AxTpXv8916WoIgQ7ahY0OzUGlDYktWYrA0irkT +Q1mT7BR5iPNIk+idyfqHcgxrVqDDFY1opYcfcS3mWm08aXFABFXcoEOUIEU4eNe9 +itg5xt8Jt1qaqQO4KBB4zb8BG1oRPjj02Bs0ec8z0gH9rJjNbUcRkEy7uVvYcOfV +r7bMxIbmdcCeKbYrDyqlaQIN4+mitF3A884saoU4dmHGSYKrUbOCprlBmCiY+2v+ +ihb/MX5UR6g83EMmqZsFt57ANEORMNQywxFa4Q== +-----END CERTIFICATE----- + No client certificate CA names sent --- SSL handshake has read 4093 bytes and written 434 bytes diff --git a/tools/testcerts/cert4.txt b/tools/testcerts/cert4.txt index 1762e35..57559e9 100644 --- a/tools/testcerts/cert4.txt +++ b/tools/testcerts/cert4.txt @@ -49,6 +49,37 @@ FceHmpqlkA2AvjdvSvwnODux3QPbMucIaJXrUUwf subject=/businessCategory=Private Organization/1.3.6.1.4.1.311.60.2.1.3=US/1.3.6.1.4.1.311.60.2.1.2=Delaware/serialNumber=3359300/street=16 Allen Rd/postalCode=03894-4801/C=US/ST=NH/L=Wolfeboro,/O=Python Software Foundation/CN=www.python.org issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert SHA2 Extended Validation Server CA --- + +Manually added intermediate certificate: +-----BEGIN CERTIFICATE----- +MIIEtjCCA56gAwIBAgIQDHmpRLCMEZUgkmFf4msdgzANBgkqhkiG9w0BAQsFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTEzMTAyMjEyMDAwMFoXDTI4MTAyMjEyMDAwMFowdTEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTE0MDIGA1UEAxMrRGlnaUNlcnQgU0hBMiBFeHRlbmRlZCBW +YWxpZGF0aW9uIFNlcnZlciBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBANdTpARR+JmmFkhLZyeqk0nQOe0MsLAAh/FnKIaFjI5j2ryxQDji0/XspQUY +uD0+xZkXMuwYjPrxDKZkIYXLBxA0sFKIKx9om9KxjxKws9LniB8f7zh3VFNfgHk/ +LhqqqB5LKw2rt2O5Nbd9FLxZS99RStKh4gzikIKHaq7q12TWmFXo/a8aUGxUvBHy +/Urynbt/DvTVvo4WiRJV2MBxNO723C3sxIclho3YIeSwTQyJ3DkmF93215SF2AQh +cJ1vb/9cuhnhRctWVyh+HA1BV6q3uCe7seT6Ku8hI3UarS2bhjWMnHe1c63YlC3k +8wyd7sFOYn4XwHGeLN7x+RAoGTMCAwEAAaOCAUkwggFFMBIGA1UdEwEB/wQIMAYB +Af8CAQAwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF +BQcDAjA0BggrBgEFBQcBAQQoMCYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRp +Z2ljZXJ0LmNvbTBLBgNVHR8ERDBCMECgPqA8hjpodHRwOi8vY3JsNC5kaWdpY2Vy +dC5jb20vRGlnaUNlcnRIaWdoQXNzdXJhbmNlRVZSb290Q0EuY3JsMD0GA1UdIAQ2 +MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5j +b20vQ1BTMB0GA1UdDgQWBBQ901Cl1qCt7vNKYApl0yHU+PjWDzAfBgNVHSMEGDAW +gBSxPsNpA/i/RwHUmCYaCALvY2QrwzANBgkqhkiG9w0BAQsFAAOCAQEAnbbQkIbh +hgLtxaDwNBx0wY12zIYKqPBKikLWP8ipTa18CK3mtlC4ohpNiAexKSHc59rGPCHg +4xFJcKx6HQGkyhE6V6t9VypAdP3THYUYUN9XR3WhfVUgLkc3UHKMf4Ib0mKPLQNa +2sPIoc4sUqIAY+tzunHISScjl2SFnjgOrWNoPLpSgVh5oywM395t6zHyuqB8bPEs +1OG9d4Q3A84ytciagRpKkk47RpqF/oOi+Z6Mo8wNXrM9zwR4jxQUezKcxwCmXMS1 +oVWNWlZopCJwqjyBcdmdqEU79OX2olHdx3ti6G8MdOu42vi/hw15UJGQmxg7kVkn +8TUoE6smftX3eg== +-----END CERTIFICATE----- + No client certificate CA names sent --- SSL handshake has read 3662 bytes and written 434 bytes diff --git a/tools/testcerts/cert5.txt b/tools/testcerts/cert5.txt index 0f3f8f1..14af5fd 100644 --- a/tools/testcerts/cert5.txt +++ b/tools/testcerts/cert5.txt @@ -67,6 +67,46 @@ uMko54p5i2QMvXtvIr/a3Nzlx6CiavI= subject=/C=US/ST=California/L=San Francisco/O=Wikimedia Foundation, Inc./CN=*.wikipedia.org issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert High Assurance CA-3 --- + +Manually added intermediate certificate: +-----BEGIN CERTIFICATE----- +MIIGWDCCBUCgAwIBAgIQCl8RTQNbF5EX0u/UA4w/OzANBgkqhkiG9w0BAQUFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTA4MDQwMjEyMDAwMFoXDTIyMDQwMzAwMDAwMFowZjEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTElMCMGA1UEAxMcRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug +Q0EtMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9hCikQH17+NDdR +CPge+yLtYb4LDXBMUGMmdRW5QYiXtvCgFbsIYOBC6AUpEIc2iihlqO8xB3RtNpcv +KEZmBMcqeSZ6mdWOw21PoF6tvD2Rwll7XjZswFPPAAgyPhBkWBATaccM7pxCUQD5 +BUTuJM56H+2MEb0SqPMV9Bx6MWkBG6fmXcCabH4JnudSREoQOiPkm7YDr6ictFuf +1EutkozOtREqqjcYjbTCuNhcBoz4/yO9NV7UfD5+gw6RlgWYw7If48hl66l7XaAs +zPw82W3tzPpLQ4zJ1LilYRyyQLYoEt+5+F/+07LJ7z20Hkt8HEyZNp496+ynaF4d +32duXvsCAwEAAaOCAvowggL2MA4GA1UdDwEB/wQEAwIBhjCCAcYGA1UdIASCAb0w +ggG5MIIBtQYLYIZIAYb9bAEDAAIwggGkMDoGCCsGAQUFBwIBFi5odHRwOi8vd3d3 +LmRpZ2ljZXJ0LmNvbS9zc2wtY3BzLXJlcG9zaXRvcnkuaHRtMIIBZAYIKwYBBQUH +AgIwggFWHoIBUgBBAG4AeQAgAHUAcwBlACAAbwBmACAAdABoAGkAcwAgAEMAZQBy +AHQAaQBmAGkAYwBhAHQAZQAgAGMAbwBuAHMAdABpAHQAdQB0AGUAcwAgAGEAYwBj +AGUAcAB0AGEAbgBjAGUAIABvAGYAIAB0AGgAZQAgAEQAaQBnAGkAQwBlAHIAdAAg +AEMAUAAvAEMAUABTACAAYQBuAGQAIAB0AGgAZQAgAFIAZQBsAHkAaQBuAGcAIABQ +AGEAcgB0AHkAIABBAGcAcgBlAGUAbQBlAG4AdAAgAHcAaABpAGMAaAAgAGwAaQBt +AGkAdAAgAGwAaQBhAGIAaQBsAGkAdAB5ACAAYQBuAGQAIABhAHIAZQAgAGkAbgBj +AG8AcgBwAG8AcgBhAHQAZQBkACAAaABlAHIAZQBpAG4AIABiAHkAIAByAGUAZgBl +AHIAZQBuAGMAZQAuMBIGA1UdEwEB/wQIMAYBAf8CAQAwNAYIKwYBBQUHAQEEKDAm +MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wgY8GA1UdHwSB +hzCBhDBAoD6gPIY6aHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0SGln +aEFzc3VyYW5jZUVWUm9vdENBLmNybDBAoD6gPIY6aHR0cDovL2NybDQuZGlnaWNl +cnQuY29tL0RpZ2lDZXJ0SGlnaEFzc3VyYW5jZUVWUm9vdENBLmNybDAfBgNVHSME +GDAWgBSxPsNpA/i/RwHUmCYaCALvY2QrwzAdBgNVHQ4EFgQUUOpzidsp+xCPnuUB +INTeeZlIg/cwDQYJKoZIhvcNAQEFBQADggEBAB7ipUiebNtTOA/vphoqrOIDQ+2a +vD6OdRvw/S4iWawTwGHi5/rpmc2HCXVUKL9GYNy+USyS8xuRfDEIcOI3ucFbqL2j +CwD7GhX9A61YasXHJJlIR0YxHpLvtF9ONMeQvzHB+LGEhtCcAarfilYGzjrpDq6X +dF3XcZpCdF/ejUN83ulV7WkAywXgemFhM9EZTfkI7qA5xSU1tyvED7Ld8aW3DiTE +JiiNeXf1L/BXunwH1OH8zVowV36GEEfdMR/X/KLCvzB8XSSq6PmuX2p0ws5rs0bY +Ib4p1I5eFdZCSucyb6Sxa1GDWL4/bcf72gMhy2oWGU4K8K2Eyl2Us1p292E= +-----END CERTIFICATE----- + + No client certificate CA names sent --- SSL handshake has read 4905 bytes and written 434 bytes -- cgit v1.1 From 574f96fba9fd01ec9725c5509f02ad13d8ab8793 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 24 Oct 2014 17:22:30 +0200 Subject: System tests for external merge --- Makefile | 33 +++++++++++++++++++++++++++++++++ test/config/frontend-1-httpd.conf | 21 +++++++++++++++++++++ test/config/frontend-1.config | 37 +++++++++++++++++++++++++++++++++++++ test/config/storage-1-httpd.conf | 21 +++++++++++++++++++++ test/config/storage-1.config | 26 ++++++++++++++++++++++++++ tools/merge.py | 2 +- tools/testcerts/roots/root1.pem | 23 +++++++++++++++++++++++ tools/testcerts/roots/root2.pem | 21 +++++++++++++++++++++ tools/testcerts/roots/root3.pem | 19 +++++++++++++++++++ 9 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 test/config/frontend-1-httpd.conf create mode 100644 test/config/frontend-1.config create mode 100644 test/config/storage-1-httpd.conf create mode 100644 test/config/storage-1.config create mode 100644 tools/testcerts/roots/root1.pem create mode 100644 tools/testcerts/roots/root2.pem create mode 100644 tools/testcerts/roots/root3.pem diff --git a/Makefile b/Makefile index aa96290..66f575f 100644 --- a/Makefile +++ b/Makefile @@ -20,3 +20,36 @@ release: printf "0" > rel/db/treesize cp -r webroot rel/catlfish test -d rel/catlfish/webroot/log || mkdir rel/catlfish/webroot/log + +tests-prepare: + -rm -r rel/known_roots + mkdir rel/known_roots + cp tools/testcerts/roots/* rel/known_roots + + mkdir -p test/nodes/frontend-1/log + mkdir -p test/nodes/storage-1/log + mkdir -p test/nodes/storage-2/log + cp test/config/frontend-1.config rel + cp test/config/frontend-1-httpd.conf rel + cp test/config/storage-1.config rel + cp test/config/storage-1-httpd.conf rel + -rm -r rel/tests + mkdir -p rel/tests/machine/machine-1/db + printf "0" > rel/tests/machine/machine-1/db/treesize + mkdir -p rel/tests/machine/machine-2/db + printf "0" > rel/tests/machine/machine-2/db/treesize + +tests-start: + (cd rel ; bin/run_erl -daemon ../test/nodes/frontend-1/ ../test/nodes/frontend-1/log/ "exec bin/erl -config frontend-1 -name frontend-1") + (cd rel ; bin/run_erl -daemon ../test/nodes/storage-1/ ../test/nodes/storage-1/log/ "exec bin/erl -config storage-1 -name storage-1") + sleep 1 + +tests-run: + (cd tools ; python testcase1.py ) || echo "Tests failed" + +tests-stop: + sleep 5 + echo "halt()." | ./rel/bin/to_erl test/nodes/frontend-1/ + echo "halt()." | ./rel/bin/to_erl test/nodes/storage-1/ + +tests: tests-prepare tests-start tests-run tests-stop diff --git a/test/config/frontend-1-httpd.conf b/test/config/frontend-1-httpd.conf new file mode 100644 index 0000000..70f91ee --- /dev/null +++ b/test/config/frontend-1-httpd.conf @@ -0,0 +1,21 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. +[ + {port, 8082}, + {bind_address, {127, 0, 0, 1}}, + {server_name, "flimsy"}, + {server_root, "catlfish/webroot"}, + {document_root, "catlfish/webroot/docroot"}, + {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head, + mod_log, mod_disk_log]}, + {erl_script_alias, {"/ct", [frontend]}}, + {erl_script_nocache, true}, + {error_log, "log/error"}, + {security_log, "log/security"}, + {transfer_log, "log/transfer"}, + {socket_type, + {essl, % See ssl(3erl) for SSL options. + [{certfile, "catlfish/webroot/certs/webcert.pem"}, + {keyfile, "catlfish/webroot/keys/webkey.pem"}, + {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}} +]. diff --git a/test/config/frontend-1.config b/test/config/frontend-1.config new file mode 100644 index 0000000..4505581 --- /dev/null +++ b/test/config/frontend-1.config @@ -0,0 +1,37 @@ +%% catlfish configuration file (-*- erlang -*-) + +[{sasl, + [{sasl_error_logger, false}, + {errlog_type, error}, + {error_logger_mf_dir, "log"}, + {error_logger_mf_maxbytes, 10485760}, % 10 MB + {error_logger_mf_maxfiles, 10}]}, + {inets, + [{services, + [{httpd, [{proplist_file, "frontend-1-httpd.conf"}]}]}]}, + {catlfish, + [{known_roots_path, "known_roots"}, + {https_servers, + [{"127.0.0.1", 8080, v1} + ]}, + {https_certfile, "catlfish/webroot/certs/webcert.pem"}, + {https_keyfile, "catlfish/webroot/keys/webkey.pem"}, + {https_cacertfile, "catlfish/webroot/certs/webcert.pem"} + ]}, + {lager, + [{handlers, + [{lager_console_backend, info}, + {lager_file_backend, [{file, "frontend-1-error.log"}, {level, error}]}, + {lager_file_backend, [{file, "frontend-1-debug.log"}, {level, debug}]}, + {lager_file_backend, [{file, "frontend-1-console.log"}, {level, info}]} + ]} + ]}, + {plop, + [{entry_root_path, "tests/machine/machine-1/db/certentries/"}, + {index_path, "tests/machine/machine-1/db/index"}, + {entryhash_root_path, "tests/machine/machine-1/db/entryhash/"}, + {treesize_path, "tests/machine/machine-1/db/treesize"}, + {indexforhash_root_path, "tests/machine/machine-1/db/certindex/"}, + {storage_nodes, ["https://127.0.0.1:8081/ct/storage/"]}, + {storage_nodes_quorum, 1} + ]}]. diff --git a/test/config/storage-1-httpd.conf b/test/config/storage-1-httpd.conf new file mode 100644 index 0000000..2f271f8 --- /dev/null +++ b/test/config/storage-1-httpd.conf @@ -0,0 +1,21 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. +[ + {port, 8081}, + {bind_address, {127, 0, 0, 1}}, + {server_name, "flimsy"}, + {server_root, "catlfish/webroot"}, + {document_root, "catlfish/webroot/docroot"}, + {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head, + mod_log, mod_disk_log]}, + {erl_script_alias, {"/ct", [storage]}}, + {erl_script_nocache, true}, + {error_log, "log/error_storage"}, + {security_log, "log/security_storage"}, + {transfer_log, "log/transfer_storage"}, + {socket_type, + {essl, % See ssl(3erl) for SSL options. + [{certfile, "catlfish/webroot/certs/webcert.pem"}, + {keyfile, "catlfish/webroot/keys/webkey.pem"}, + {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}} +]. diff --git a/test/config/storage-1.config b/test/config/storage-1.config new file mode 100644 index 0000000..5b67bbf --- /dev/null +++ b/test/config/storage-1.config @@ -0,0 +1,26 @@ +%% catlfish configuration file (-*- erlang -*-) + +[{sasl, + [{sasl_error_logger, false}, + {errlog_type, error}, + {error_logger_mf_dir, "log"}, + {error_logger_mf_maxbytes, 10485760}, % 10 MB + {error_logger_mf_maxfiles, 10}]}, + {inets, + [{services, + [{httpd, [{proplist_file, "storage-1-httpd.conf"}]}]}]}, + {lager, + [{handlers, + [{lager_console_backend, info}, + {lager_file_backend, [{file, "storage-1-error.log"}, {level, error}]}, + {lager_file_backend, [{file, "storage-1-debug.log"}, {level, debug}]}, + {lager_file_backend, [{file, "storage-1-console.log"}, {level, info}]} + ]} + ]}, + {plop, + [{entry_root_path, "tests/machine/machine-1/db/certentries/"}, + {index_path, "tests/machine/machine-1/db/index"}, + {newentries_path, "tests/machine/machine-1/db/newentries"}, + {entryhash_root_path, "tests/machine/machine-1/db/entryhash/"}, + {treesize_path, "tests/machine/machine-1/db/treesize"}, + {indexforhash_root_path, "tests/machine/machine-1/db/certindex/"}]}]. diff --git a/tools/merge.py b/tools/merge.py index 7120d04..14466ff 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -10,7 +10,7 @@ import urllib import urllib2 import sys -frontendnodes = ["https://127.0.0.1:8080/"] +frontendnodes = ["https://127.0.0.1:8082/"] storagenodes = ["https://127.0.0.1:8081/"] chainsdir = "../rel/mergedb/chains" diff --git a/tools/testcerts/roots/root1.pem b/tools/testcerts/roots/root1.pem new file mode 100644 index 0000000..e077900 --- /dev/null +++ b/tools/testcerts/roots/root1.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCBlzELMAkGA1UE +BhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEeMBwGA1UEChMVVGhl +IFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAd +BgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgx +OTIyWjCBlzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0 +eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8vd3d3LnVz +ZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdhcmUwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlI +wrthdBKWHTxqctU8EGc6Oe0rE81m65UJM6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFd +tqdt++BxF2uiiPsA3/4aMXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8 +i4fDidNdoI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqIDsjf +Pe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9KsyoUhbAgMBAAGjgbkw +gbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFKFyXyYbKJhDlV0HN9WF +lp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNF +UkZpcnN0LUhhcmR3YXJlLmNybDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUF +BwMGBggrBgEFBQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM +//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28GpgoiskliCE7/yMgUsogW +XecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gECJChicsZUN/KHAG8HQQZexB2 +lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kn +iCrVWFCVH/A7HFe7fRQ5YiuayZSSKqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67 +nfhmqA== +-----END CERTIFICATE----- diff --git a/tools/testcerts/roots/root2.pem b/tools/testcerts/roots/root2.pem new file mode 100644 index 0000000..bdb6474 --- /dev/null +++ b/tools/testcerts/roots/root2.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIENjCCAx6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBvMQswCQYDVQQGEwJTRTEUMBIGA1UEChML +QWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYD +VQQDExlBZGRUcnVzdCBFeHRlcm5hbCBDQSBSb290MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEw +NDgzOFowbzELMAkGA1UEBhMCU0UxFDASBgNVBAoTC0FkZFRydXN0IEFCMSYwJAYDVQQLEx1BZGRU +cnVzdCBFeHRlcm5hbCBUVFAgTmV0d29yazEiMCAGA1UEAxMZQWRkVHJ1c3QgRXh0ZXJuYWwgQ0Eg +Um9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf3GjPm8gAELTngTlvtH7xsD821 ++iO2zt6bETOXpClMfZOfvUq8k+0DGuOPz+VtUFrWlymUWoCwSXrbLpX9uMq/NzgtHj6RQa1wVsfw +Tz/oMp50ysiQVOnGXw94nZpAPA6sYapeFI+eh6FqUNzXmk6vBbOmcZSccbNQYArHE504B4YCqOmo +aSYYkKtMsE8jqzpPhNjfzp/haW+710LXa0Tkx63ubUFfclpxCDezeWWkWaCUN/cALw3CknLa0Dhy +2xSoRcRdKn23tNbE7qzNE0S3ySvdQwAl+mG5aWpYIxG3pzOPVnVZ9c0p10a3CitlttNCbxWyuHv7 +7+ldU9U0WicCAwEAAaOB3DCB2TAdBgNVHQ4EFgQUrb2YejS0Jvf6xCZU7wO94CTLVBowCwYDVR0P +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wgZkGA1UdIwSBkTCBjoAUrb2YejS0Jvf6xCZU7wO94CTL +VBqhc6RxMG8xCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRk +VHJ1c3QgRXh0ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB +IFJvb3SCAQEwDQYJKoZIhvcNAQEFBQADggEBALCb4IUlwtYj4g+WBpKdQZic2YR5gdkeWxQHIzZl +j7DYd7usQWxHYINRsPkyPef89iYTx4AWpb9a/IfPeHmJIZriTAcKhjW88t5RxNKWt9x+Tu5w/Rw5 +6wwCURQtjr0W4MHfRnXnJK3s9EK0hZNwEGe6nQY1ShjTK3rMUUKhemPR5ruhxSvCNr4TDea9Y355 +e6cJDUCrat2PisP29owaQgVR1EX1n6diIWgVIEM8med8vSTYqZEXc4g/VhsxOBi0cQ+azcgOno4u +G+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5amnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ= +-----END CERTIFICATE----- diff --git a/tools/testcerts/roots/root3.pem b/tools/testcerts/roots/root3.pem new file mode 100644 index 0000000..81c8a7d --- /dev/null +++ b/tools/testcerts/roots/root3.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBsMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSsw +KQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5jZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAw +MFoXDTMxMTExMDAwMDAwMFowbDELMAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZ +MBcGA1UECxMQd3d3LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFu +Y2UgRVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm+9S75S0t +Mqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTWPNt0OKRKzE0lgvdKpVMS +OO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEMxChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3 +MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFBIk5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQ +NAQTXKFx01p8VdteZOE3hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUe +h10aUAsgEsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMB +Af8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaAFLE+w2kD+L9HAdSY +JhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3NecnzyIZgYIVyHbIUf4KmeqvxgydkAQ +V8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6zeM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFp +myPInngiK3BD41VHMWEZ71jFhS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkK +mNEVX58Svnw2Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe +vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep+OkuE6N36B9K +-----END CERTIFICATE----- -- cgit v1.1 From f0c78cf6c6caedffd1069b4c41e41ec8ca7fb1c7 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 25 Oct 2014 02:08:25 +0200 Subject: Move internal HTTP APIs to mochiweb. --- catlfish.config | 2 +- src/catlfish_sup.erl | 5 +++-- src/catlfish_web.erl | 11 +++++------ test/config/frontend-1-httpd.conf | 21 --------------------- test/config/frontend-1.config | 6 ++---- test/config/storage-1-httpd.conf | 21 --------------------- test/config/storage-1.config | 11 ++++++++--- 7 files changed, 19 insertions(+), 58 deletions(-) delete mode 100644 test/config/frontend-1-httpd.conf delete mode 100644 test/config/storage-1-httpd.conf diff --git a/catlfish.config b/catlfish.config index 672f997..91868e5 100644 --- a/catlfish.config +++ b/catlfish.config @@ -10,7 +10,7 @@ {catlfish, [{known_roots_path, "known_roots"}, {https_servers, - [{"127.0.0.1", 8080, v1} + [{external_https_api, "127.0.0.1", 8080, v1} ]}, {https_certfile, "catlfish/webroot/certs/webcert.pem"}, {https_keyfile, "catlfish/webroot/keys/webkey.pem"}, diff --git a/src/catlfish_sup.erl b/src/catlfish_sup.erl index abdac44..0b6c306 100644 --- a/src/catlfish_sup.erl +++ b/src/catlfish_sup.erl @@ -16,7 +16,7 @@ init([]) -> {cacertfile, application:get_env(catlfish, https_cacertfile, none)}], Servers = lists:map(fun (Config) -> - {IpAddress, Port, Module} = Config, + {ChildName, IpAddress, Port, Module} = Config, {ok, IPv4Address} = inet:parse_ipv4strict_address(IpAddress), WebConfig = [{ip, IPv4Address}, @@ -24,11 +24,12 @@ init([]) -> {ssl, true}, {ssl_opts, SSLOptions} ], - {catlfish_web, + {ChildName, {catlfish_web, start, [WebConfig, Module]}, permanent, 5000, worker, dynamic} end, application:get_env(catlfish, https_servers, [])), + lager:debug("Starting servers ~p", [Servers]), {ok, {{one_for_one, 3, 10}, Servers}}. diff --git a/src/catlfish_web.erl b/src/catlfish_web.erl index cdc1a39..f3231e4 100644 --- a/src/catlfish_web.erl +++ b/src/catlfish_web.erl @@ -2,16 +2,14 @@ %%% See LICENSE for licensing information. -module(catlfish_web). --export([start/2, stop/0, loop/2]). +-export([start/2, loop/2]). start(Options, Module) -> + lager:debug("Starting catlfish web server: ~p", [Module]), Loop = fun (Req) -> ?MODULE:loop(Req, Module) end, - mochiweb_http:start([{name, ?MODULE}, {loop, Loop} | Options]). - -stop() -> - mochiweb_http:stop(?MODULE). + mochiweb_http:start([{name, Module}, {loop, Loop} | Options]). loop(Req, Module) -> "/" ++ Path = Req:get(path), @@ -42,7 +40,8 @@ loop(Req, Module) -> end catch Type:What -> - lager:error("Crash in ~p for path ~p: ~p ~n~p~n~p~n", [Module, Path, Type, What, erlang:get_stacktrace()]), + [CrashFunction | Stack] = erlang:get_stacktrace(), + lager:error("Crash in ~p for path ~p: ~p ~p~n~p~n~p~n", [Module, Path, Type, What, CrashFunction, Stack]), Req:respond({500, [{"Content-Type", "text/plain"}], "Internal Server Error\n"}) end. diff --git a/test/config/frontend-1-httpd.conf b/test/config/frontend-1-httpd.conf deleted file mode 100644 index 70f91ee..0000000 --- a/test/config/frontend-1-httpd.conf +++ /dev/null @@ -1,21 +0,0 @@ -%%% Copyright (c) 2014, NORDUnet A/S. -%%% See LICENSE for licensing information. -[ - {port, 8082}, - {bind_address, {127, 0, 0, 1}}, - {server_name, "flimsy"}, - {server_root, "catlfish/webroot"}, - {document_root, "catlfish/webroot/docroot"}, - {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head, - mod_log, mod_disk_log]}, - {erl_script_alias, {"/ct", [frontend]}}, - {erl_script_nocache, true}, - {error_log, "log/error"}, - {security_log, "log/security"}, - {transfer_log, "log/transfer"}, - {socket_type, - {essl, % See ssl(3erl) for SSL options. - [{certfile, "catlfish/webroot/certs/webcert.pem"}, - {keyfile, "catlfish/webroot/keys/webkey.pem"}, - {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}} -]. diff --git a/test/config/frontend-1.config b/test/config/frontend-1.config index 4505581..79d887d 100644 --- a/test/config/frontend-1.config +++ b/test/config/frontend-1.config @@ -6,13 +6,11 @@ {error_logger_mf_dir, "log"}, {error_logger_mf_maxbytes, 10485760}, % 10 MB {error_logger_mf_maxfiles, 10}]}, - {inets, - [{services, - [{httpd, [{proplist_file, "frontend-1-httpd.conf"}]}]}]}, {catlfish, [{known_roots_path, "known_roots"}, {https_servers, - [{"127.0.0.1", 8080, v1} + [{external_https_api, "127.0.0.1", 8080, v1}, + {frontend_https_api, "127.0.0.1", 8082, frontend} ]}, {https_certfile, "catlfish/webroot/certs/webcert.pem"}, {https_keyfile, "catlfish/webroot/keys/webkey.pem"}, diff --git a/test/config/storage-1-httpd.conf b/test/config/storage-1-httpd.conf deleted file mode 100644 index 2f271f8..0000000 --- a/test/config/storage-1-httpd.conf +++ /dev/null @@ -1,21 +0,0 @@ -%%% Copyright (c) 2014, NORDUnet A/S. -%%% See LICENSE for licensing information. -[ - {port, 8081}, - {bind_address, {127, 0, 0, 1}}, - {server_name, "flimsy"}, - {server_root, "catlfish/webroot"}, - {document_root, "catlfish/webroot/docroot"}, - {modules, [mod_alias, mod_auth, mod_esi, mod_get, mod_head, - mod_log, mod_disk_log]}, - {erl_script_alias, {"/ct", [storage]}}, - {erl_script_nocache, true}, - {error_log, "log/error_storage"}, - {security_log, "log/security_storage"}, - {transfer_log, "log/transfer_storage"}, - {socket_type, - {essl, % See ssl(3erl) for SSL options. - [{certfile, "catlfish/webroot/certs/webcert.pem"}, - {keyfile, "catlfish/webroot/keys/webkey.pem"}, - {cacertfile, "catlfish/webroot/certs/webcert.pem"}]}} -]. diff --git a/test/config/storage-1.config b/test/config/storage-1.config index 5b67bbf..b176e1f 100644 --- a/test/config/storage-1.config +++ b/test/config/storage-1.config @@ -6,9 +6,14 @@ {error_logger_mf_dir, "log"}, {error_logger_mf_maxbytes, 10485760}, % 10 MB {error_logger_mf_maxfiles, 10}]}, - {inets, - [{services, - [{httpd, [{proplist_file, "storage-1-httpd.conf"}]}]}]}, + {catlfish, + [{https_servers, + [{storage_https_api, "127.0.0.1", 8081, storage} + ]}, + {https_certfile, "catlfish/webroot/certs/webcert.pem"}, + {https_keyfile, "catlfish/webroot/keys/webkey.pem"}, + {https_cacertfile, "catlfish/webroot/certs/webcert.pem"} + ]}, {lager, [{handlers, [{lager_console_backend, info}, -- cgit v1.1 From 602d063427b5d808478ad558f99e5efe28a2cc6e Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 25 Oct 2014 02:10:17 +0200 Subject: Stop using jiffy --- ebin/catlfish.app | 2 +- src/catlfish.erl | 39 +++++------ src/v1.erl | 189 ++++++++++++++++++++++++++---------------------------- 3 files changed, 108 insertions(+), 122 deletions(-) diff --git a/ebin/catlfish.app b/ebin/catlfish.app index beea7d6..75bc876 100644 --- a/ebin/catlfish.app +++ b/ebin/catlfish.app @@ -8,5 +8,5 @@ [{description, "catlfish -- Certificate Transparency Log Server"}, {vsn, "0.2.0-dev"}, {modules, [v1, catlfish_app]}, - {applications, [kernel, stdlib, plop, inets, jiffy, lager, mochiweb]}, + {applications, [kernel, stdlib, plop, inets, lager, mochiweb]}, {mod, {catlfish_app, []}}]}. diff --git a/src/catlfish.erl b/src/catlfish.erl index 73066bb..6cc741d 100644 --- a/src/catlfish.erl +++ b/src/catlfish.erl @@ -98,13 +98,11 @@ add_chain(LeafCert, CertChain) -> plop:spt(list_to_binary([<>, serialise_signature_type(certificate_timestamp), serialise(TimestampedEntry)])), - binary_to_list( - jiffy:encode( - {[{sct_version, ?PROTOCOL_VERSION}, - {id, base64:encode(plop:get_logid())}, - {timestamp, TimestampedEntry#timestamped_entry.timestamp}, - {extensions, base64:encode(<<>>)}, - {signature, base64:encode(plop:serialise(SCT_sig))}]})). + {[{sct_version, ?PROTOCOL_VERSION}, + {id, base64:encode(plop:get_logid())}, + {timestamp, TimestampedEntry#timestamped_entry.timestamp}, + {extensions, base64:encode(<<>>)}, + {signature, base64:encode(plop:serialise(SCT_sig))}]}. -spec serialise_logentry(integer(), binary(), [binary()]) -> binary(). serialise_logentry(Timestamp, LeafCert, CertChain) -> @@ -118,24 +116,21 @@ serialise_logentry(Timestamp, LeafCert, CertChain) -> -spec entries(non_neg_integer(), non_neg_integer()) -> list(). entries(Start, End) -> - binary_to_list( - jiffy:encode({[{entries, x_entries(plop:get(Start, End))}]})). + {[{entries, x_entries(plop:get(Start, End))}]}. -spec entry_and_proof(non_neg_integer(), non_neg_integer()) -> list(). entry_and_proof(Index, TreeSize) -> - binary_to_list( - jiffy:encode( - case plop:inclusion_and_entry(Index, TreeSize) of - {ok, Entry, Path} -> - {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry), - MTL = build_mtl(Timestamp, LeafCertVector), - {[{leaf_input, base64:encode(MTL)}, - {extra_data, base64:encode(CertChainVector)}, - {audit_path, [base64:encode(X) || X <- Path]}]}; - {notfound, Msg} -> - {[{success, false}, - {error_message, list_to_binary(Msg)}]} - end)). + case plop:inclusion_and_entry(Index, TreeSize) of + {ok, Entry, Path} -> + {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry), + MTL = build_mtl(Timestamp, LeafCertVector), + {[{leaf_input, base64:encode(MTL)}, + {extra_data, base64:encode(CertChainVector)}, + {audit_path, [base64:encode(X) || X <- Path]}]}; + {notfound, Msg} -> + {[{success, false}, + {error_message, list_to_binary(Msg)}]} + end. %% Private functions. unpack_entry(Entry) -> diff --git a/src/v1.erl b/src/v1.erl index 086fa35..d9796fa 100644 --- a/src/v1.erl +++ b/src/v1.erl @@ -9,32 +9,31 @@ %% Public functions, i.e. part of URL. request(post, "ct/v1/add-chain", Input) -> - R = case (catch jiffy:decode(Input)) of - {error, E} -> - html("add-chain: bad input:", E); - {[{<<"chain">>, ChainBase64}]} -> - case (catch [base64:decode(X) || X <- ChainBase64]) of - {'EXIT', _} -> - html("add-chain: invalid base64-encoded chain: ", - [ChainBase64]); - [LeafCert | CertChain] -> - Roots = catlfish:known_roots(), - case x509:normalise_chain(Roots, [LeafCert|CertChain]) of - {ok, [Leaf | Chain]} -> - io:format("[info] adding ~p~n", - [x509:cert_string(LeafCert)]), - success(catlfish:add_chain(Leaf, Chain)); - {error, Reason} -> - io:format("[info] rejecting ~p: ~p~n", - [x509:cert_string(LeafCert), Reason]), - html("add-chain: invalid chain", Reason) - end; - Invalid -> - html("add-chain: chain is not a list: ", [Invalid]) - end; - _ -> html("add-chain: missing input: chain", Input) - end, - R; + case (catch mochijson2:decode(Input)) of + {error, E} -> + html("add-chain: bad input:", E); + {struct, [{<<"chain">>, ChainBase64}]} -> + case (catch [base64:decode(X) || X <- ChainBase64]) of + {'EXIT', _} -> + html("add-chain: invalid base64-encoded chain: ", + [ChainBase64]); + [LeafCert | CertChain] -> + Roots = catlfish:known_roots(), + case x509:normalise_chain(Roots, [LeafCert|CertChain]) of + {ok, [Leaf | Chain]} -> + lager:info("adding ~p", + [x509:cert_string(LeafCert)]), + success(catlfish:add_chain(Leaf, Chain)); + {error, Reason} -> + lager:info("rejecting ~p: ~p", + [x509:cert_string(LeafCert), Reason]), + html("add-chain: invalid chain", Reason) + end; + Invalid -> + html("add-chain: chain is not a list: ", [Invalid]) + end; + _ -> html("add-chain: missing input: chain", Input) + end; request(post, "ct/v1/add-pre-chain", _Input) -> niy(); @@ -49,91 +48,83 @@ request(get, "ct/v1/get-sth", _Query) -> {sha256_root_hash, base64:encode(Roothash)}, {tree_head_signature, base64:encode( plop:serialise(Signature))}], - success(jiffy:encode({R})); + success({R}); request(get, "ct/v1/get-sth-consistency", Query) -> - R = case lists:sort(Query) of - [{"first", FirstInput}, {"second", SecondInput}] -> - {First, _} = string:to_integer(FirstInput), - {Second, _} = string:to_integer(SecondInput), - case lists:member(error, [First, Second]) of - true -> - html("get-sth-consistency: bad input:", - [FirstInput, SecondInput]); - false -> - success( - jiffy:encode( - {[{consistency, - [base64:encode(X) || - X <- plop:consistency(First, Second)]}]})) - end; - _ -> html("get-sth-consistency: bad input:", Query) - end, - R; + case lists:sort(Query) of + [{"first", FirstInput}, {"second", SecondInput}] -> + {First, _} = string:to_integer(FirstInput), + {Second, _} = string:to_integer(SecondInput), + case lists:member(error, [First, Second]) of + true -> + html("get-sth-consistency: bad input:", + [FirstInput, SecondInput]); + false -> + success( + {[{consistency, + [base64:encode(X) || + X <- plop:consistency(First, Second)]}]}) + end; + _ -> html("get-sth-consistency: bad input:", Query) + end; request(get, "ct/v1/get-proof-by-hash", Query) -> - R = case lists:sort(Query) of - [{"hash", HashInput}, {"tree_size", TreeSizeInput}] -> - Hash = case (catch base64:decode(HashInput)) of - {'EXIT', _} -> error; - H -> H - end, - {TreeSize, _} = string:to_integer(TreeSizeInput), - case lists:member(error, [Hash, TreeSize]) of - true -> - html("get-proof-by-hash: bad input:", - [HashInput, TreeSizeInput]); - false -> - success( - jiffy:encode( - case plop:inclusion(Hash, TreeSize) of - {ok, Index, Path} -> - {[{leaf_index, Index}, - {audit_path, - [base64:encode(X) || X <- Path]}]}; - {notfound, Msg} -> - %% FIXME: http status 400 - {[{success, false}, - {error_message, list_to_binary(Msg)}]} - end)) - end; - _ -> html("get-proof-by-hash: bad input:", Query) - end, - R; + case lists:sort(Query) of + [{"hash", HashInput}, {"tree_size", TreeSizeInput}] -> + Hash = case (catch base64:decode(HashInput)) of + {'EXIT', _} -> error; + H -> H + end, + {TreeSize, _} = string:to_integer(TreeSizeInput), + case lists:member(error, [Hash, TreeSize]) of + true -> + html("get-proof-by-hash: bad input:", + [HashInput, TreeSizeInput]); + false -> + case plop:inclusion(Hash, TreeSize) of + {ok, Index, Path} -> + success({[{leaf_index, Index}, + {audit_path, + [base64:encode(X) || X <- Path]}]}); + {notfound, Msg} -> + html("get-proof-by-hash: hash not found", Msg) + end + end; + _ -> html("get-proof-by-hash: bad input:", Query) + end; request(get, "ct/v1/get-entries", Query) -> - R = case lists:sort(Query) of - [{"end", EndInput}, {"start", StartInput}] -> - {Start, _} = string:to_integer(StartInput), - {End, _} = string:to_integer(EndInput), - case lists:member(error, [Start, End]) of - true -> html("get-entries: bad input:", [Start, End]); - false -> success(catlfish:entries(Start, min(End, Start + 999))) - end; - _ -> html("get-entries: bad input:", Query) - end, - R; + case lists:sort(Query) of + [{"end", EndInput}, {"start", StartInput}] -> + {Start, _} = string:to_integer(StartInput), + {End, _} = string:to_integer(EndInput), + case lists:member(error, [Start, End]) of + true -> html("get-entries: bad input:", [Start, End]); + false -> success( + catlfish:entries(Start, min(End, Start + 999))) + end; + _ -> html("get-entries: bad input:", Query) + end; request(get, "ct/v1/get-entry-and-proof", Query) -> - R = case lists:sort(Query) of - [{"leaf_index", IndexInput}, {"tree_size", TreeSizeInput}] -> - {Index, _} = string:to_integer(IndexInput), - {TreeSize, _} = string:to_integer(TreeSizeInput), - case lists:member(error, [Index, TreeSize]) of - true -> - html("get-entry-and-proof: not integers: ", - [IndexInput, TreeSizeInput]); - false -> success(catlfish:entry_and_proof(Index, TreeSize)) - end; - _ -> html("get-entry-and-proof: bad input:", Query) - end, - R; + case lists:sort(Query) of + [{"leaf_index", IndexInput}, {"tree_size", TreeSizeInput}] -> + {Index, _} = string:to_integer(IndexInput), + {TreeSize, _} = string:to_integer(TreeSizeInput), + case lists:member(error, [Index, TreeSize]) of + true -> + html("get-entry-and-proof: not integers: ", + [IndexInput, TreeSizeInput]); + false -> success(catlfish:entry_and_proof(Index, TreeSize)) + end; + _ -> html("get-entry-and-proof: bad input:", Query) + end; request(get, "ct/v1/get-roots", _Query) -> R = [{certificates, [base64:encode(Der) || Der <- catlfish:update_known_roots()]}], - success(jiffy:encode({R})); + success({R}); request(_Method, _Path, _) -> none. @@ -151,4 +142,4 @@ niy() -> html("NIY - Not Implemented Yet|", []). success(Data) -> - {200, [{"Content-Type", "text/json"}], Data}. + {200, [{"Content-Type", "text/json"}], mochijson2:encode(Data)}. -- cgit v1.1 From acb3fca6727fe2eb346cac0f48906901c17a89c9 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 26 Oct 2014 00:36:57 +0200 Subject: Rewrite root certificate cache handling --- src/catlfish.erl | 35 +++++++++++++++++------------------ src/catlfish.hrl | 4 ++++ src/catlfish_app.erl | 9 +++++++++ 3 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 src/catlfish.hrl diff --git a/src/catlfish.erl b/src/catlfish.erl index 6cc741d..5d96278 100644 --- a/src/catlfish.erl +++ b/src/catlfish.erl @@ -5,6 +5,7 @@ -export([add_chain/2, entries/2, entry_and_proof/2]). -export([known_roots/0, update_known_roots/0]). -include_lib("eunit/include/eunit.hrl"). +-include("catlfish.hrl"). -define(PROTOCOL_VERSION, 0). @@ -159,7 +160,7 @@ decode_tls_vector(Binary, LengthLen) -> <> = Rest, {ExtractedBinary, Rest2}. --define(ROOTS_TABLE, catlfish_roots). +-define(ROOTS_CACHE_KEY, roots). update_known_roots() -> case application:get_env(catlfish, known_roots_path) of @@ -178,22 +179,20 @@ known_roots() -> -spec known_roots(file:filename(), use_cache|update_tab) -> list(). known_roots(Directory, CacheUsage) -> - case ets:info(?ROOTS_TABLE) of - undefined -> - read_pemfiles_from_dir( - ets:new(?ROOTS_TABLE, [set, protected, named_table]), - Directory); - _ -> - case CacheUsage of - use_cache -> - ets:lookup_element(?ROOTS_TABLE, list, 2); - update_tab -> - read_pemfiles_from_dir(?ROOTS_TABLE, Directory) - end + case CacheUsage of + use_cache -> + case ets:lookup(?CACHE_TABLE, ?ROOTS_CACHE_KEY) of + [] -> + read_pemfiles_from_dir(Directory); + [{roots, DerList}] -> + DerList + end; + update_tab -> + read_pemfiles_from_dir(Directory) end. --spec read_pemfiles_from_dir(ets:tab(), file:filename()) -> list(). -read_pemfiles_from_dir(Tab, Dir) -> +-spec read_pemfiles_from_dir(file:filename()) -> list(). +read_pemfiles_from_dir(Dir) -> DerList = case file:list_dir(Dir) of {error, enoent} -> @@ -208,7 +207,7 @@ read_pemfiles_from_dir(Tab, Dir) -> Filenames), ders_from_pemfiles(Dir, Files) end, - true = ets:insert(Tab, {list, DerList}), + true = ets:insert(?CACHE_TABLE, {?ROOTS_CACHE_KEY, DerList}), DerList. ders_from_pemfiles(Dir, Filenames) -> @@ -251,7 +250,7 @@ read_pemfiles_test_() -> fun() -> {known_roots(?PEMFILES_DIR_OK, use_cache), known_roots(?PEMFILES_DIR_OK, use_cache)} end, - fun(_) -> ets:delete(?ROOTS_TABLE) end, + fun(_) -> ets:delete(?CACHE_TABLE, ?ROOTS_CACHE_KEY) end, fun({L, LCached}) -> [?_assertMatch(7, length(L)), ?_assertEqual(L, LCached)] @@ -260,5 +259,5 @@ read_pemfiles_test_() -> read_pemfiles_fail_test_() -> {setup, fun() -> known_roots(?PEMFILES_DIR_NONEXISTENT, use_cache) end, - fun(_) -> ets:delete(?ROOTS_TABLE) end, + fun(_) -> ets:delete(?CACHE_TABLE, ?ROOTS_CACHE_KEY) end, fun(Empty) -> [?_assertMatch([], Empty)] end}. diff --git a/src/catlfish.hrl b/src/catlfish.hrl new file mode 100644 index 0000000..46e882b --- /dev/null +++ b/src/catlfish.hrl @@ -0,0 +1,4 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. + +-define(CACHE_TABLE, catlfish_cache). diff --git a/src/catlfish_app.erl b/src/catlfish_app.erl index cfb55cd..e24a1bb 100644 --- a/src/catlfish_app.erl +++ b/src/catlfish_app.erl @@ -8,11 +8,20 @@ %% Application callbacks -export([start/2, stop/1]). +-include("catlfish.hrl"). + %% =================================================================== %% Application callbacks %% =================================================================== start(normal, Args) -> + case ets:info(?CACHE_TABLE) of + undefined -> + ok; + _ -> + ets:delete(?CACHE_TABLE) + end, + ets:new(?CACHE_TABLE, [set, public, named_table]), catlfish_sup:start_link(Args). stop(_State) -> -- cgit v1.1 From 7fd299ab23beee422789f679a35c9526c54fb1fb Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 14:31:20 +0100 Subject: Handle missing entries in merge --- tools/merge.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tools/merge.py b/tools/merge.py index 14466ff..3aeecdf 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -79,6 +79,22 @@ def sendlog(baseurl, submission): print "========================" raise e +def sendentry(baseurl, entry, hash): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendentry", + json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)})).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendentry", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print hash + print "======= RESPONSE =======" + print result + print "========================" + raise e + def sendsth(baseurl, submission): try: result = urllib2.urlopen(baseurl + "ct/frontend/sendsth", @@ -113,6 +129,8 @@ certsinlog = set(logorder) new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] +print "adding entries" +added_entries = 0 for new_entry in new_entries: hash = base64.b64decode(new_entry["hash"]) entry = base64.b64decode(new_entry["entry"]) @@ -121,13 +139,19 @@ for new_entry in new_entries: add_to_logorder(hash) logorder.append(hash) certsinlog.add(hash) - print "added", base64.b16encode(hash) + added_entries += 1 +print "added", added_entries, "entries" for frontendnode in frontendnodes: + print "distributing for node", frontendnode curpos = get_curpos(frontendnode) + print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] sendlog(frontendnode, {"start": curpos, "hashes": entries}) + print "log sent" missingentries = get_missingentries(frontendnode) print "missing entries:", missingentries - # XXX: no test case for missing entries yet, waiting to implement + for missingentry in missingentries: + hash = base64.b64decode(missingentry) + sendentry(frontendnode, read_chain(hash), hash) sendsth(frontendnode, {"tree_size": len(logorder)}) -- cgit v1.1 From 91e5b7f4b85cdbc399ccaa1bb1d813e0d829f3d5 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 14:37:01 +0100 Subject: submitcert.py: submit multiple cert chains --- tools/certtools.py | 14 +++++++ tools/submitcert.py | 104 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 78 insertions(+), 40 deletions(-) diff --git a/tools/certtools.py b/tools/certtools.py index 8d64ee4..7b901cf 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -10,6 +10,7 @@ import struct import sys import hashlib import ecdsa +import datetime publickeys = { "https://ct.googleapis.com/pilot/": @@ -204,3 +205,16 @@ def get_leaf_hash(merkle_tree_leaf): leaf_hash.update(merkle_tree_leaf) return leaf_hash.digest() + +def timing_point(timer_dict=None, name=None): + t = datetime.datetime.now() + if timer_dict: + starttime = timer_dict["lasttime"] + stoptime = t + deltatime = stoptime - starttime + timer_dict["deltatimes"].append((name, deltatime.seconds * 1000000 + deltatime.microseconds)) + timer_dict["lasttime"] = t + return None + else: + timer_dict = {"deltatimes":[], "lasttime":t} + return timer_dict diff --git a/tools/submitcert.py b/tools/submitcert.py index 4f1609c..80a3e37 100755 --- a/tools/submitcert.py +++ b/tools/submitcert.py @@ -12,63 +12,87 @@ import struct import hashlib import itertools from certtools import * +import os + +from multiprocessing import Pool baseurl = sys.argv[1] -certfile = sys.argv[2] +certfilepath = sys.argv[2] + +lookup_in_log = False +check_sig = False + +if certfilepath[-1] == "/": + certfiles = [certfilepath + filename for filename in sorted(os.listdir(certfilepath))] +else: + certfiles = [certfilepath] + +def submitcert(certfile): + timing = timing_point() + certs = get_certs_from_file(certfile) + timing_point(timing, "readcerts") + + result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)}) + + timing_point(timing, "addchain") + + try: + if check_sig: + check_sct_signature(baseurl, certs[0], result) + timing_point(timing, "checksig") + except AssertionError, e: + print "ERROR:", e + sys.exit(1) + except ecdsa.keys.BadSignatureError, e: + print "ERROR: bad signature" + sys.exit(1) -lookup_in_log = True + if lookup_in_log: -certs = get_certs_from_file(certfile) + merkle_tree_leaf = pack_mtl(result["timestamp"], certs[0]) -result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)}) + leaf_hash = get_leaf_hash(merkle_tree_leaf) -try: - check_sct_signature(baseurl, certs[0], result) -except AssertionError, e: - print "ERROR:", e - sys.exit(1) -except ecdsa.keys.BadSignatureError, e: - print "ERROR: bad signature" - sys.exit(1) -print "signature check succeeded" + sth = get_sth(baseurl) -if lookup_in_log: + proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"]) - merkle_tree_leaf = pack_mtl(result["timestamp"], certs[0]) + leaf_index = proof["leaf_index"] - leaf_hash = get_leaf_hash(merkle_tree_leaf) + entries = get_entries(baseurl, leaf_index, leaf_index) - sth = get_sth(baseurl) + fetched_entry = entries["entries"][0] - proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"]) + print "does the leaf_input of the fetched entry match what we calculated:", \ + base64.decodestring(fetched_entry["leaf_input"]) == merkle_tree_leaf - leaf_index = proof["leaf_index"] + extra_data = fetched_entry["extra_data"] - entries = get_entries(baseurl, leaf_index, leaf_index) + certchain = decode_certificate_chain(base64.decodestring(extra_data)) - fetched_entry = entries["entries"][0] + submittedcertchain = certs[1:] - print "does the leaf_input of the fetched entry match what we calculated:", \ - base64.decodestring(fetched_entry["leaf_input"]) == merkle_tree_leaf + for (submittedcert, fetchedcert, i) in zip(submittedcertchain, + certchain, itertools.count(1)): + print "cert", i, "in chain is the same:", submittedcert == fetchedcert - extra_data = fetched_entry["extra_data"] + if len(certchain) == len(submittedcertchain) + 1: + last_issuer = get_cert_info(certs[-1])["issuer"] + root_subject = get_cert_info(certchain[-1])["subject"] + print "issuer of last cert in submitted chain and " \ + "subject of last cert in fetched chain is the same:", \ + last_issuer == root_subject + elif len(certchain) == len(submittedcertchain): + print "cert chains are the same length" + else: + print "ERROR: fetched cert chain has length", len(certchain), + print "and submitted chain has length", len(submittedcertchain) - certchain = decode_certificate_chain(base64.decodestring(extra_data)) + timing_point(timing, "lookup") + return timing["deltatimes"] - submittedcertchain = certs[1:] +p = Pool(1) - for (submittedcert, fetchedcert, i) in zip(submittedcertchain, - certchain, itertools.count(1)): - print "cert", i, "in chain is the same:", submittedcert == fetchedcert +for timing in p.imap_unordered(submitcert, certfiles): + print timing - if len(certchain) == len(submittedcertchain) + 1: - last_issuer = get_cert_info(certs[-1])["issuer"] - root_subject = get_cert_info(certchain[-1])["subject"] - print "issuer of last cert in submitted chain and " \ - "subject of last cert in fetched chain is the same:", \ - last_issuer == root_subject - elif len(certchain) == len(submittedcertchain): - print "cert chains are the same length" - else: - print "ERROR: fetched cert chain has length", len(certchain), - print "and submitted chain has length", len(submittedcertchain) -- cgit v1.1 From 44f48b0f96aba0009bd43036eea443f07cec71b9 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 14:37:48 +0100 Subject: Added fetchallcerts.py --- tools/certtools.py | 8 ++++++++ tools/fetchallcerts.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tools/fetchallcerts.py diff --git a/tools/certtools.py b/tools/certtools.py index 7b901cf..16c2105 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -199,6 +199,14 @@ def pack_mtl(timestamp, leafcert): merkle_tree_leaf = version + leaf_type + timestamped_entry return merkle_tree_leaf +def unpack_mtl(merkle_tree_leaf): + version = merkle_tree_leaf[0:1] + leaf_type = merkle_tree_leaf[1:2] + timestamped_entry = merkle_tree_leaf[2:] + (timestamp, entry_type) = struct.unpack(">QH", timestamped_entry[0:10]) + (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3) + return (leafcert, timestamp) + def get_leaf_hash(merkle_tree_leaf): leaf_hash = hashlib.sha256() leaf_hash.update(struct.pack(">b", 0)) diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py new file mode 100644 index 0000000..801e296 --- /dev/null +++ b/tools/fetchallcerts.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import urllib2 +import urllib +import json +import base64 +import sys +import struct +import hashlib +import itertools +from certtools import * + +def extract_original_entry(entry): + leaf_input = base64.decodestring(entry["leaf_input"]) + (leaf_cert, timestamp) = unpack_mtl(leaf_input) + extra_data = base64.decodestring(entry["extra_data"]) + certchain = decode_certificate_chain(extra_data) + return [leaf_cert] + certchain + +def get_entries_wrapper(baseurl, start, end): + fetched_entries = [] + while len(fetched_entries) < (end - start + 1): + print "fetching from", start + len(fetched_entries) + entries = get_entries(baseurl, start + len(fetched_entries), end)["entries"] + if len(entries) == 0: + break + fetched_entries.extend(entries) + return fetched_entries + +baseurl = sys.argv[1] +destination_directory = sys.argv[2] + +sth = get_sth(baseurl) +tree_size = sth["tree_size"] + +print tree_size + +entries = get_entries_wrapper(baseurl, 0, tree_size) + +print len(entries) + +for entry, i in zip(entries, range(0, len(entries))): + chain = extract_original_entry(entry) + f = open(destination_directory + "/" + ("%06d" % i), "w") + for cert in chain: + print >> f, "-----BEGIN CERTIFICATE-----" + print >> f, base64.encodestring(cert).rstrip() + print >> f, "-----END CERTIFICATE-----" + print >> f, "" -- cgit v1.1 From b9c709204da83be2f315664f9f263c6890b1bc8d Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 16:13:41 +0100 Subject: fetchallcerts.py: calculate root hash --- tools/certtools.py | 24 +++++++++++++++++++++++ tools/fetchallcerts.py | 52 +++++++++++++++++++++++++++++++++++--------------- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/tools/certtools.py b/tools/certtools.py index 16c2105..b132caa 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -226,3 +226,27 @@ def timing_point(timer_dict=None, name=None): else: timer_dict = {"deltatimes":[], "lasttime":t} return timer_dict + +def internal_hash(pair): + if len(pair) == 1: + return pair[0] + else: + hash = hashlib.sha256() + hash.update(struct.pack(">b", 1)) + hash.update(pair[0]) + hash.update(pair[1]) + return hash.digest() + +def chunks(l, n): + return [l[i:i+n] for i in range(0, len(l), n)] + +def next_merkle_layer(layer): + return [internal_hash(pair) for pair in chunks(layer, 2)] + +def build_merkle_tree(layer0): + layers = [] + current_layer = layer0 + while len(current_layer) > 1: + current_layer = next_merkle_layer(current_layer) + layers.append(current_layer) + return layers diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py index 801e296..dad5241 100644 --- a/tools/fetchallcerts.py +++ b/tools/fetchallcerts.py @@ -4,6 +4,7 @@ # Copyright (c) 2014, NORDUnet A/S. # See LICENSE for licensing information. +import argparse import urllib2 import urllib import json @@ -14,6 +15,11 @@ import hashlib import itertools from certtools import * +parser = argparse.ArgumentParser(description='') +parser.add_argument('baseurl', help="Base URL for CT server") +parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir') +args = parser.parse_args() + def extract_original_entry(entry): leaf_input = base64.decodestring(entry["leaf_input"]) (leaf_cert, timestamp) = unpack_mtl(leaf_input) @@ -23,7 +29,7 @@ def extract_original_entry(entry): def get_entries_wrapper(baseurl, start, end): fetched_entries = [] - while len(fetched_entries) < (end - start + 1): + while start + len(fetched_entries) < (end + 1): print "fetching from", start + len(fetched_entries) entries = get_entries(baseurl, start + len(fetched_entries), end)["entries"] if len(entries) == 0: @@ -31,23 +37,39 @@ def get_entries_wrapper(baseurl, start, end): fetched_entries.extend(entries) return fetched_entries -baseurl = sys.argv[1] -destination_directory = sys.argv[2] +def print_layer(layer): + for entry in layer: + print base64.b16encode(entry) -sth = get_sth(baseurl) +sth = get_sth(args.baseurl) tree_size = sth["tree_size"] +root_hash = base64.decodestring(sth["sha256_root_hash"]) + +print "tree size", tree_size +print "root hash", base64.b16encode(root_hash) + +entries = get_entries_wrapper(args.baseurl, 0, tree_size - 1) + +print "fetched", len(entries), "entries" + +layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries] + +tree = build_merkle_tree(layer0) -print tree_size +calculated_root_hash = tree[-1][0] -entries = get_entries_wrapper(baseurl, 0, tree_size) +print "calculated root hash", base64.b16encode(calculated_root_hash) -print len(entries) +if calculated_root_hash != root_hash: + print "fetched root hash and calculated root hash different, aborting" + sys.exit(1) -for entry, i in zip(entries, range(0, len(entries))): - chain = extract_original_entry(entry) - f = open(destination_directory + "/" + ("%06d" % i), "w") - for cert in chain: - print >> f, "-----BEGIN CERTIFICATE-----" - print >> f, base64.encodestring(cert).rstrip() - print >> f, "-----END CERTIFICATE-----" - print >> f, "" +if args.store: + for entry, i in zip(entries, range(0, len(entries))): + chain = extract_original_entry(entry) + f = open(args.store + "/" + ("%06d" % i), "w") + for cert in chain: + print >> f, "-----BEGIN CERTIFICATE-----" + print >> f, base64.encodestring(cert).rstrip() + print >> f, "-----END CERTIFICATE-----" + print >> f, "" -- cgit v1.1 From 89b8a59da38daf465a934f039cc7cf3efaf1468e Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 16:56:03 +0100 Subject: merge.py: send whole sth in sendsth call --- tools/certtools.py | 20 ++++++++++++++++++++ tools/merge.py | 27 ++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/tools/certtools.py b/tools/certtools.py index b132caa..0ce8885 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -143,6 +143,11 @@ def decode_signature(signature): assert rest == "" return (hash_alg, signature_alg, unpacked_signature) +def encode_signature(hash_alg, signature_alg, unpacked_signature): + signature = struct.pack(">bb", hash_alg, signature_alg) + signature += tls_array(unpacked_signature, 2) + return signature + def check_signature(baseurl, signature, data): publickey = base64.decodestring(publickeys[baseurl]) (hash_alg, signature_alg, unpacked_signature) = decode_signature(signature) @@ -155,6 +160,12 @@ def check_signature(baseurl, signature, data): vk.verify(unpacked_signature, data, hashfunc=hashlib.sha256, sigdecode=ecdsa.util.sigdecode_der) +def create_signature(privatekey, data): + sk = ecdsa.SigningKey.from_der(privatekey) + unpacked_signature = sk.sign(data, hashfunc=hashlib.sha256, + sigencode=ecdsa.util.sigencode_der) + return encode_signature(4, 3, unpacked_signature) + def check_sth_signature(baseurl, sth): signature = base64.decodestring(sth["tree_head_signature"]) @@ -167,6 +178,15 @@ def check_sth_signature(baseurl, sth): check_signature(baseurl, signature, tree_head) +def create_sth_signature(tree_size, timestamp, root_hash, privatekey): + version = struct.pack(">b", 0) + signature_type = struct.pack(">b", 1) + timestamp_packed = struct.pack(">Q", timestamp) + tree_size_packed = struct.pack(">Q", tree_size) + tree_head = version + signature_type + timestamp_packed + tree_size_packed + root_hash + + return create_signature(privatekey, tree_head) + def check_sct_signature(baseurl, leafcert, sct): publickey = base64.decodestring(publickeys[baseurl]) calculated_logid = hashlib.sha256(publickey).digest() diff --git a/tools/merge.py b/tools/merge.py index 3aeecdf..7c6e357 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -9,7 +9,10 @@ import base64 import urllib import urllib2 import sys +import time +from certtools import build_merkle_tree, create_sth_signature, check_sth_signature +ctbaseurl = "https://127.0.0.1:8080/" frontendnodes = ["https://127.0.0.1:8082/"] storagenodes = ["https://127.0.0.1:8081/"] @@ -142,6 +145,28 @@ for new_entry in new_entries: added_entries += 1 print "added", added_entries, "entries" +tree = build_merkle_tree(logorder) +tree_size = len(logorder) +root_hash = tree[-1][0] +timestamp = int(time.time() * 1000) +privatekey = base64.decodestring( + "MHcCAQEEIMM/FjZ4FSzfENTTwGpTve6CP+IVr" + "Y7p8OKV634uJI/foAoGCCqGSM49AwEHoUQDQg" + "AE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9PMS5l" + "qoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJ" + "GolozZYmNHE2kQ==") + +tree_head_signature = create_sth_signature(tree_size, timestamp, + root_hash, privatekey) + +sth = {"tree_size": tree_size, "timestamp": timestamp, + "sha256_root_hash": base64.b64encode(root_hash), + "tree_head_signature": base64.b64encode(tree_head_signature)} + +check_sth_signature(ctbaseurl, sth) + +print "root hash", base64.b16encode(root_hash) + for frontendnode in frontendnodes: print "distributing for node", frontendnode curpos = get_curpos(frontendnode) @@ -154,4 +179,4 @@ for frontendnode in frontendnodes: for missingentry in missingentries: hash = base64.b64decode(missingentry) sendentry(frontendnode, read_chain(hash), hash) - sendsth(frontendnode, {"tree_size": len(logorder)}) + sendsth(frontendnode, sth) -- cgit v1.1 From 8de65ab10cbdf6ea5509f321f025686e6ab0ecaf Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 28 Oct 2014 16:03:11 +0100 Subject: certtools.py: fix bug in build_merkle_tree --- tools/certtools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/certtools.py b/tools/certtools.py index 0ce8885..cbb4ff7 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -264,8 +264,11 @@ def next_merkle_layer(layer): return [internal_hash(pair) for pair in chunks(layer, 2)] def build_merkle_tree(layer0): + if len(layer0) == 0: + return [[hashlib.sha256().digest()]] layers = [] current_layer = layer0 + layers.append(current_layer) while len(current_layer) > 1: current_layer = next_merkle_layer(current_layer) layers.append(current_layer) -- cgit v1.1 From ce6fac1d34f71732e56d5b4e3f4819d6fe282957 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 28 Oct 2014 16:04:06 +0100 Subject: Check return value from merge.py --- tools/testcase1.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tools/testcase1.py b/tools/testcase1.py index dd3597e..639cd69 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -122,11 +122,15 @@ def get_and_check_entry(timestamp, chain, leaf_index): print_and_check_tree_size(0) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) + testgroup("cert1") result1 = do_add_chain(cc1) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(1) @@ -134,7 +138,8 @@ result2 = do_add_chain(cc1) assert_equal(result2["timestamp"], result1["timestamp"], "timestamp") -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(1) @@ -147,7 +152,8 @@ testgroup("cert2") result3 = do_add_chain(cc2) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(2) @@ -158,7 +164,8 @@ testgroup("cert3") result4 = do_add_chain(cc3) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(3) @@ -170,7 +177,8 @@ testgroup("cert4") result5 = do_add_chain(cc4) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(4) @@ -183,7 +191,8 @@ testgroup("cert5") result6 = do_add_chain(cc5) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(5) -- cgit v1.1 From 87e02103ea3f47b825b415c415f7d2940d009b42 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 28 Oct 2014 16:06:11 +0100 Subject: httpd.conf removed, reflect this in Makefile. Touch test db files. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 66f575f..0239a6c 100644 --- a/Makefile +++ b/Makefile @@ -30,14 +30,14 @@ tests-prepare: mkdir -p test/nodes/storage-1/log mkdir -p test/nodes/storage-2/log cp test/config/frontend-1.config rel - cp test/config/frontend-1-httpd.conf rel cp test/config/storage-1.config rel - cp test/config/storage-1-httpd.conf rel -rm -r rel/tests mkdir -p rel/tests/machine/machine-1/db printf "0" > rel/tests/machine/machine-1/db/treesize mkdir -p rel/tests/machine/machine-2/db printf "0" > rel/tests/machine/machine-2/db/treesize + touch rel/tests/machine/machine-1/db/index + touch rel/tests/machine/machine-1/db/newentries tests-start: (cd rel ; bin/run_erl -daemon ../test/nodes/frontend-1/ ../test/nodes/frontend-1/log/ "exec bin/erl -config frontend-1 -name frontend-1") -- cgit v1.1