From 194c9aa3b8c463fa487dc9ef7e172332a8d94d72 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 20 Oct 2014 14:33:41 +0200 Subject: Added external merging support --- tools/merge.py | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++ tools/testcase1.py | 13 ++++++ 2 files changed, 146 insertions(+) create mode 100755 tools/merge.py (limited to 'tools') diff --git a/tools/merge.py b/tools/merge.py new file mode 100755 index 0000000..7120d04 --- /dev/null +++ b/tools/merge.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Kungliga Tekniska Högskolan +# (KTH Royal Institute of Technology, Stockholm, Sweden). +# See LICENSE for licensing information. + +import json +import base64 +import urllib +import urllib2 +import sys + +frontendnodes = ["https://127.0.0.1:8080/"] +storagenodes = ["https://127.0.0.1:8081/"] + +chainsdir = "../rel/mergedb/chains" +logorderfile = "../rel/mergedb/logorder" + +def parselogrow(row): + return base64.b16decode(row) + +def get_logorder(): + f = open(logorderfile, "r") + return [parselogrow(row.rstrip()) for row in f] + +def write_chain(key, value): + f = open(chainsdir + "/" + base64.b16encode(key), "w") + f.write(value) + f.close() + +def read_chain(key): + f = open(chainsdir + "/" + base64.b16encode(key), "r") + value = f.read() + f.close() + return value + +def add_to_logorder(key): + f = open(logorderfile, "a") + f.write(base64.b16encode(key) + "\n") + f.close() + +def get_new_entries(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/storage/fetchnewentries").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"entries"] + print "ERROR: fetchnewentries", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: fetchnewentries", e.read() + sys.exit(1) + +def get_curpos(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/currentposition").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"position"] + print "ERROR: currentposition", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: currentposition", e.read() + sys.exit(1) + +def sendlog(baseurl, submission): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendlog", + json.dumps(submission)).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendlog", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e + +def sendsth(baseurl, submission): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendsth", + json.dumps(submission)).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendsth", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e + +def get_missingentries(baseurl): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/missingentries").read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"entries"] + print "ERROR: missingentries", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: missingentries", e.read() + sys.exit(1) + + +logorder = get_logorder() +certsinlog = set(logorder) + +new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] + +for new_entry in new_entries: + hash = base64.b64decode(new_entry["hash"]) + entry = base64.b64decode(new_entry["entry"]) + if hash not in certsinlog: + write_chain(hash, entry) + add_to_logorder(hash) + logorder.append(hash) + certsinlog.add(hash) + print "added", base64.b16encode(hash) + +for frontendnode in frontendnodes: + curpos = get_curpos(frontendnode) + entries = [base64.b64encode(entry) for entry in logorder[curpos:]] + sendlog(frontendnode, {"start": curpos, "hashes": entries}) + missingentries = get_missingentries(frontendnode) + print "missing entries:", missingentries + # XXX: no test case for missing entries yet, waiting to implement + sendsth(frontendnode, {"tree_size": len(logorder)}) diff --git a/tools/testcase1.py b/tools/testcase1.py index eab6c6f..2d5e0e8 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Copyright (c) 2014, NORDUnet A/S. # See LICENSE for licensing information. @@ -125,12 +126,16 @@ testgroup("cert1") result1 = do_add_chain(cc1) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(1) result2 = do_add_chain(cc1) assert_equal(result2["timestamp"], result1["timestamp"], "timestamp") +subprocess.call(["./merge.py"]) + print_and_check_tree_size(1) # TODO: add invalid cert and check that it generates an error @@ -142,6 +147,8 @@ testgroup("cert2") result3 = do_add_chain(cc2) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(2) get_and_validate_proof(result1["timestamp"], cc1, 0, 1) @@ -151,6 +158,8 @@ testgroup("cert3") result4 = do_add_chain(cc3) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(3) get_and_validate_proof(result1["timestamp"], cc1, 0, 2) @@ -161,6 +170,8 @@ testgroup("cert4") result5 = do_add_chain(cc4) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(4) get_and_validate_proof(result1["timestamp"], cc1, 0, 2) @@ -172,6 +183,8 @@ testgroup("cert5") result6 = do_add_chain(cc5) +subprocess.call(["./merge.py"]) + print_and_check_tree_size(5) get_and_validate_proof(result1["timestamp"], cc1, 0, 3) -- cgit v1.1 From bda18b837a255aeb20366c736272698fa3223c4f Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 24 Oct 2014 15:23:33 +0200 Subject: Repair tests to work with x509 validation code. Add intermediate certificates to test chains. --- tools/testcase1.py | 2 +- tools/testcerts/cert3.txt | 30 ++++++++++++++++++++++++++++++ tools/testcerts/cert4.txt | 31 +++++++++++++++++++++++++++++++ tools/testcerts/cert5.txt | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testcase1.py b/tools/testcase1.py index 2d5e0e8..dd3597e 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -105,7 +105,7 @@ def get_and_check_entry(timestamp, chain, leaf_index): assert_equal(fetchedcert, submittedcert, "cert %d in chain" % (i,)) if len(certchain) == len(submittedcertchain) + 1: - last_issuer = get_cert_info(certs[-1])["issuer"] + last_issuer = get_cert_info(submittedcertchain[-1])["issuer"] root_subject = get_cert_info(certchain[-1])["subject"] if last_issuer == root_subject: print_success("fetched chain has an appended root cert") diff --git a/tools/testcerts/cert3.txt b/tools/testcerts/cert3.txt index d12e485..a776b58 100644 --- a/tools/testcerts/cert3.txt +++ b/tools/testcerts/cert3.txt @@ -38,6 +38,36 @@ SbDmRK4Rxa5UmgfZnezD0snHVUCrzKzP subject=/OU=Domain Control Validated/CN=*.nordu.net issuer=/C=NL/O=TERENA/CN=TERENA SSL CA --- + +Manually added intermediate certificate: +-----BEGIN CERTIFICATE----- +MIIEmDCCA4CgAwIBAgIQS8gUAy8H+mqk8Nop32F5ujANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNMDkwNTE4MDAwMDAwWhcNMjAwNTMwMTA0ODM4WjA2MQswCQYD +VQQGEwJOTDEPMA0GA1UEChMGVEVSRU5BMRYwFAYDVQQDEw1URVJFTkEgU1NMIENB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw+NIxC9cwcupmf0booNd +ij2tOtDipEMfTQ7+NSUwpWkbxOjlwY9UfuFqoppcXN49/ALOlrhfj4NbzGBAkPjk +tjolnF8UUeyx56+eUKExVccCvaxSin81joL6hK0V/qJ/gxA6VVOULAEWdJRUYyij +8lspPZSIgCDiFFkhGbSkmOFg5vLrooCDQ+CtaPN5GYtoQ1E/iptBhQw1jF218bbl +p8ODtWsjb9Sl61DllPFKX+4nSxQSFSRMDc9ijbcAIa06Mg9YC18em9HfnY6pGTVQ +L0GprTvG4EWyUzl/Ib8iGodcNK5Sbwd9ogtOnyt5pn0T3fV/g3wvWl13eHiRoBS/ +fQIDAQABo4IBPjCCATowHwYDVR0jBBgwFoAUoXJfJhsomEOVXQc31YWWnUvSw0Uw +HQYDVR0OBBYEFAy9k2gM896ro0lrKzdXR+qQ47ntMA4GA1UdDwEB/wQEAwIBBjAS +BgNVHRMBAf8ECDAGAQH/AgEAMBgGA1UdIAQRMA8wDQYLKwYBBAGyMQECAh0wRAYD +VR0fBD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VS +Rmlyc3QtSGFyZHdhcmUuY3JsMHQGCCsGAQUFBwEBBGgwZjA9BggrBgEFBQcwAoYx +aHR0cDovL2NydC51c2VydHJ1c3QuY29tL1VUTkFkZFRydXN0U2VydmVyX0NBLmNy +dDAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AudXNlcnRydXN0LmNvbTANBgkqhkiG +9w0BAQUFAAOCAQEATiPuSJz2hYtxxApuc5NywDqOgIrZs8qy1AGcKM/yXA4hRJML +thoh45gBlA5nSYEevj0NTmDa76AxTpXv8916WoIgQ7ahY0OzUGlDYktWYrA0irkT +Q1mT7BR5iPNIk+idyfqHcgxrVqDDFY1opYcfcS3mWm08aXFABFXcoEOUIEU4eNe9 +itg5xt8Jt1qaqQO4KBB4zb8BG1oRPjj02Bs0ec8z0gH9rJjNbUcRkEy7uVvYcOfV +r7bMxIbmdcCeKbYrDyqlaQIN4+mitF3A884saoU4dmHGSYKrUbOCprlBmCiY+2v+ +ihb/MX5UR6g83EMmqZsFt57ANEORMNQywxFa4Q== +-----END CERTIFICATE----- + No client certificate CA names sent --- SSL handshake has read 4093 bytes and written 434 bytes diff --git a/tools/testcerts/cert4.txt b/tools/testcerts/cert4.txt index 1762e35..57559e9 100644 --- a/tools/testcerts/cert4.txt +++ b/tools/testcerts/cert4.txt @@ -49,6 +49,37 @@ FceHmpqlkA2AvjdvSvwnODux3QPbMucIaJXrUUwf subject=/businessCategory=Private Organization/1.3.6.1.4.1.311.60.2.1.3=US/1.3.6.1.4.1.311.60.2.1.2=Delaware/serialNumber=3359300/street=16 Allen Rd/postalCode=03894-4801/C=US/ST=NH/L=Wolfeboro,/O=Python Software Foundation/CN=www.python.org issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert SHA2 Extended Validation Server CA --- + +Manually added intermediate certificate: +-----BEGIN CERTIFICATE----- +MIIEtjCCA56gAwIBAgIQDHmpRLCMEZUgkmFf4msdgzANBgkqhkiG9w0BAQsFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTEzMTAyMjEyMDAwMFoXDTI4MTAyMjEyMDAwMFowdTEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTE0MDIGA1UEAxMrRGlnaUNlcnQgU0hBMiBFeHRlbmRlZCBW +YWxpZGF0aW9uIFNlcnZlciBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBANdTpARR+JmmFkhLZyeqk0nQOe0MsLAAh/FnKIaFjI5j2ryxQDji0/XspQUY +uD0+xZkXMuwYjPrxDKZkIYXLBxA0sFKIKx9om9KxjxKws9LniB8f7zh3VFNfgHk/ +LhqqqB5LKw2rt2O5Nbd9FLxZS99RStKh4gzikIKHaq7q12TWmFXo/a8aUGxUvBHy +/Urynbt/DvTVvo4WiRJV2MBxNO723C3sxIclho3YIeSwTQyJ3DkmF93215SF2AQh +cJ1vb/9cuhnhRctWVyh+HA1BV6q3uCe7seT6Ku8hI3UarS2bhjWMnHe1c63YlC3k +8wyd7sFOYn4XwHGeLN7x+RAoGTMCAwEAAaOCAUkwggFFMBIGA1UdEwEB/wQIMAYB +Af8CAQAwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF +BQcDAjA0BggrBgEFBQcBAQQoMCYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRp +Z2ljZXJ0LmNvbTBLBgNVHR8ERDBCMECgPqA8hjpodHRwOi8vY3JsNC5kaWdpY2Vy +dC5jb20vRGlnaUNlcnRIaWdoQXNzdXJhbmNlRVZSb290Q0EuY3JsMD0GA1UdIAQ2 +MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5j +b20vQ1BTMB0GA1UdDgQWBBQ901Cl1qCt7vNKYApl0yHU+PjWDzAfBgNVHSMEGDAW +gBSxPsNpA/i/RwHUmCYaCALvY2QrwzANBgkqhkiG9w0BAQsFAAOCAQEAnbbQkIbh +hgLtxaDwNBx0wY12zIYKqPBKikLWP8ipTa18CK3mtlC4ohpNiAexKSHc59rGPCHg +4xFJcKx6HQGkyhE6V6t9VypAdP3THYUYUN9XR3WhfVUgLkc3UHKMf4Ib0mKPLQNa +2sPIoc4sUqIAY+tzunHISScjl2SFnjgOrWNoPLpSgVh5oywM395t6zHyuqB8bPEs +1OG9d4Q3A84ytciagRpKkk47RpqF/oOi+Z6Mo8wNXrM9zwR4jxQUezKcxwCmXMS1 +oVWNWlZopCJwqjyBcdmdqEU79OX2olHdx3ti6G8MdOu42vi/hw15UJGQmxg7kVkn +8TUoE6smftX3eg== +-----END CERTIFICATE----- + No client certificate CA names sent --- SSL handshake has read 3662 bytes and written 434 bytes diff --git a/tools/testcerts/cert5.txt b/tools/testcerts/cert5.txt index 0f3f8f1..14af5fd 100644 --- a/tools/testcerts/cert5.txt +++ b/tools/testcerts/cert5.txt @@ -67,6 +67,46 @@ uMko54p5i2QMvXtvIr/a3Nzlx6CiavI= subject=/C=US/ST=California/L=San Francisco/O=Wikimedia Foundation, Inc./CN=*.wikipedia.org issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert High Assurance CA-3 --- + +Manually added intermediate certificate: +-----BEGIN CERTIFICATE----- +MIIGWDCCBUCgAwIBAgIQCl8RTQNbF5EX0u/UA4w/OzANBgkqhkiG9w0BAQUFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTA4MDQwMjEyMDAwMFoXDTIyMDQwMzAwMDAwMFowZjEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTElMCMGA1UEAxMcRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug +Q0EtMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9hCikQH17+NDdR +CPge+yLtYb4LDXBMUGMmdRW5QYiXtvCgFbsIYOBC6AUpEIc2iihlqO8xB3RtNpcv +KEZmBMcqeSZ6mdWOw21PoF6tvD2Rwll7XjZswFPPAAgyPhBkWBATaccM7pxCUQD5 +BUTuJM56H+2MEb0SqPMV9Bx6MWkBG6fmXcCabH4JnudSREoQOiPkm7YDr6ictFuf +1EutkozOtREqqjcYjbTCuNhcBoz4/yO9NV7UfD5+gw6RlgWYw7If48hl66l7XaAs +zPw82W3tzPpLQ4zJ1LilYRyyQLYoEt+5+F/+07LJ7z20Hkt8HEyZNp496+ynaF4d +32duXvsCAwEAAaOCAvowggL2MA4GA1UdDwEB/wQEAwIBhjCCAcYGA1UdIASCAb0w +ggG5MIIBtQYLYIZIAYb9bAEDAAIwggGkMDoGCCsGAQUFBwIBFi5odHRwOi8vd3d3 +LmRpZ2ljZXJ0LmNvbS9zc2wtY3BzLXJlcG9zaXRvcnkuaHRtMIIBZAYIKwYBBQUH +AgIwggFWHoIBUgBBAG4AeQAgAHUAcwBlACAAbwBmACAAdABoAGkAcwAgAEMAZQBy +AHQAaQBmAGkAYwBhAHQAZQAgAGMAbwBuAHMAdABpAHQAdQB0AGUAcwAgAGEAYwBj +AGUAcAB0AGEAbgBjAGUAIABvAGYAIAB0AGgAZQAgAEQAaQBnAGkAQwBlAHIAdAAg +AEMAUAAvAEMAUABTACAAYQBuAGQAIAB0AGgAZQAgAFIAZQBsAHkAaQBuAGcAIABQ +AGEAcgB0AHkAIABBAGcAcgBlAGUAbQBlAG4AdAAgAHcAaABpAGMAaAAgAGwAaQBt +AGkAdAAgAGwAaQBhAGIAaQBsAGkAdAB5ACAAYQBuAGQAIABhAHIAZQAgAGkAbgBj +AG8AcgBwAG8AcgBhAHQAZQBkACAAaABlAHIAZQBpAG4AIABiAHkAIAByAGUAZgBl +AHIAZQBuAGMAZQAuMBIGA1UdEwEB/wQIMAYBAf8CAQAwNAYIKwYBBQUHAQEEKDAm +MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wgY8GA1UdHwSB +hzCBhDBAoD6gPIY6aHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0SGln +aEFzc3VyYW5jZUVWUm9vdENBLmNybDBAoD6gPIY6aHR0cDovL2NybDQuZGlnaWNl +cnQuY29tL0RpZ2lDZXJ0SGlnaEFzc3VyYW5jZUVWUm9vdENBLmNybDAfBgNVHSME +GDAWgBSxPsNpA/i/RwHUmCYaCALvY2QrwzAdBgNVHQ4EFgQUUOpzidsp+xCPnuUB +INTeeZlIg/cwDQYJKoZIhvcNAQEFBQADggEBAB7ipUiebNtTOA/vphoqrOIDQ+2a +vD6OdRvw/S4iWawTwGHi5/rpmc2HCXVUKL9GYNy+USyS8xuRfDEIcOI3ucFbqL2j +CwD7GhX9A61YasXHJJlIR0YxHpLvtF9ONMeQvzHB+LGEhtCcAarfilYGzjrpDq6X +dF3XcZpCdF/ejUN83ulV7WkAywXgemFhM9EZTfkI7qA5xSU1tyvED7Ld8aW3DiTE +JiiNeXf1L/BXunwH1OH8zVowV36GEEfdMR/X/KLCvzB8XSSq6PmuX2p0ws5rs0bY +Ib4p1I5eFdZCSucyb6Sxa1GDWL4/bcf72gMhy2oWGU4K8K2Eyl2Us1p292E= +-----END CERTIFICATE----- + + No client certificate CA names sent --- SSL handshake has read 4905 bytes and written 434 bytes -- cgit v1.1 From 574f96fba9fd01ec9725c5509f02ad13d8ab8793 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 24 Oct 2014 17:22:30 +0200 Subject: System tests for external merge --- tools/merge.py | 2 +- tools/testcerts/roots/root1.pem | 23 +++++++++++++++++++++++ tools/testcerts/roots/root2.pem | 21 +++++++++++++++++++++ tools/testcerts/roots/root3.pem | 19 +++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 tools/testcerts/roots/root1.pem create mode 100644 tools/testcerts/roots/root2.pem create mode 100644 tools/testcerts/roots/root3.pem (limited to 'tools') diff --git a/tools/merge.py b/tools/merge.py index 7120d04..14466ff 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -10,7 +10,7 @@ import urllib import urllib2 import sys -frontendnodes = ["https://127.0.0.1:8080/"] +frontendnodes = ["https://127.0.0.1:8082/"] storagenodes = ["https://127.0.0.1:8081/"] chainsdir = "../rel/mergedb/chains" diff --git a/tools/testcerts/roots/root1.pem b/tools/testcerts/roots/root1.pem new file mode 100644 index 0000000..e077900 --- /dev/null +++ b/tools/testcerts/roots/root1.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCBlzELMAkGA1UE +BhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEeMBwGA1UEChMVVGhl +IFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAd +BgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgx +OTIyWjCBlzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0 +eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8vd3d3LnVz +ZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdhcmUwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlI +wrthdBKWHTxqctU8EGc6Oe0rE81m65UJM6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFd +tqdt++BxF2uiiPsA3/4aMXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8 +i4fDidNdoI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqIDsjf +Pe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9KsyoUhbAgMBAAGjgbkw +gbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFKFyXyYbKJhDlV0HN9WF +lp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNF +UkZpcnN0LUhhcmR3YXJlLmNybDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUF +BwMGBggrBgEFBQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM +//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28GpgoiskliCE7/yMgUsogW +XecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gECJChicsZUN/KHAG8HQQZexB2 +lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kn +iCrVWFCVH/A7HFe7fRQ5YiuayZSSKqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67 +nfhmqA== +-----END CERTIFICATE----- diff --git a/tools/testcerts/roots/root2.pem b/tools/testcerts/roots/root2.pem new file mode 100644 index 0000000..bdb6474 --- /dev/null +++ b/tools/testcerts/roots/root2.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIENjCCAx6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBvMQswCQYDVQQGEwJTRTEUMBIGA1UEChML +QWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYD +VQQDExlBZGRUcnVzdCBFeHRlcm5hbCBDQSBSb290MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEw +NDgzOFowbzELMAkGA1UEBhMCU0UxFDASBgNVBAoTC0FkZFRydXN0IEFCMSYwJAYDVQQLEx1BZGRU +cnVzdCBFeHRlcm5hbCBUVFAgTmV0d29yazEiMCAGA1UEAxMZQWRkVHJ1c3QgRXh0ZXJuYWwgQ0Eg +Um9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf3GjPm8gAELTngTlvtH7xsD821 ++iO2zt6bETOXpClMfZOfvUq8k+0DGuOPz+VtUFrWlymUWoCwSXrbLpX9uMq/NzgtHj6RQa1wVsfw +Tz/oMp50ysiQVOnGXw94nZpAPA6sYapeFI+eh6FqUNzXmk6vBbOmcZSccbNQYArHE504B4YCqOmo +aSYYkKtMsE8jqzpPhNjfzp/haW+710LXa0Tkx63ubUFfclpxCDezeWWkWaCUN/cALw3CknLa0Dhy +2xSoRcRdKn23tNbE7qzNE0S3ySvdQwAl+mG5aWpYIxG3pzOPVnVZ9c0p10a3CitlttNCbxWyuHv7 +7+ldU9U0WicCAwEAAaOB3DCB2TAdBgNVHQ4EFgQUrb2YejS0Jvf6xCZU7wO94CTLVBowCwYDVR0P +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wgZkGA1UdIwSBkTCBjoAUrb2YejS0Jvf6xCZU7wO94CTL +VBqhc6RxMG8xCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRk +VHJ1c3QgRXh0ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB +IFJvb3SCAQEwDQYJKoZIhvcNAQEFBQADggEBALCb4IUlwtYj4g+WBpKdQZic2YR5gdkeWxQHIzZl +j7DYd7usQWxHYINRsPkyPef89iYTx4AWpb9a/IfPeHmJIZriTAcKhjW88t5RxNKWt9x+Tu5w/Rw5 +6wwCURQtjr0W4MHfRnXnJK3s9EK0hZNwEGe6nQY1ShjTK3rMUUKhemPR5ruhxSvCNr4TDea9Y355 +e6cJDUCrat2PisP29owaQgVR1EX1n6diIWgVIEM8med8vSTYqZEXc4g/VhsxOBi0cQ+azcgOno4u +G+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5amnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ= +-----END CERTIFICATE----- diff --git a/tools/testcerts/roots/root3.pem b/tools/testcerts/roots/root3.pem new file mode 100644 index 0000000..81c8a7d --- /dev/null +++ b/tools/testcerts/roots/root3.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBsMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSsw +KQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5jZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAw +MFoXDTMxMTExMDAwMDAwMFowbDELMAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZ +MBcGA1UECxMQd3d3LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFu +Y2UgRVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm+9S75S0t +Mqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTWPNt0OKRKzE0lgvdKpVMS +OO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEMxChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3 +MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFBIk5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQ +NAQTXKFx01p8VdteZOE3hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUe +h10aUAsgEsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMB +Af8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaAFLE+w2kD+L9HAdSY +JhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3NecnzyIZgYIVyHbIUf4KmeqvxgydkAQ +V8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6zeM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFp +myPInngiK3BD41VHMWEZ71jFhS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkK +mNEVX58Svnw2Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe +vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep+OkuE6N36B9K +-----END CERTIFICATE----- -- cgit v1.1 From 7fd299ab23beee422789f679a35c9526c54fb1fb Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 14:31:20 +0100 Subject: Handle missing entries in merge --- tools/merge.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/merge.py b/tools/merge.py index 14466ff..3aeecdf 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -79,6 +79,22 @@ def sendlog(baseurl, submission): print "========================" raise e +def sendentry(baseurl, entry, hash): + try: + result = urllib2.urlopen(baseurl + "ct/frontend/sendentry", + json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)})).read() + return json.loads(result) + except urllib2.HTTPError, e: + print "ERROR: sendentry", e.read() + sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print hash + print "======= RESPONSE =======" + print result + print "========================" + raise e + def sendsth(baseurl, submission): try: result = urllib2.urlopen(baseurl + "ct/frontend/sendsth", @@ -113,6 +129,8 @@ certsinlog = set(logorder) new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] +print "adding entries" +added_entries = 0 for new_entry in new_entries: hash = base64.b64decode(new_entry["hash"]) entry = base64.b64decode(new_entry["entry"]) @@ -121,13 +139,19 @@ for new_entry in new_entries: add_to_logorder(hash) logorder.append(hash) certsinlog.add(hash) - print "added", base64.b16encode(hash) + added_entries += 1 +print "added", added_entries, "entries" for frontendnode in frontendnodes: + print "distributing for node", frontendnode curpos = get_curpos(frontendnode) + print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] sendlog(frontendnode, {"start": curpos, "hashes": entries}) + print "log sent" missingentries = get_missingentries(frontendnode) print "missing entries:", missingentries - # XXX: no test case for missing entries yet, waiting to implement + for missingentry in missingentries: + hash = base64.b64decode(missingentry) + sendentry(frontendnode, read_chain(hash), hash) sendsth(frontendnode, {"tree_size": len(logorder)}) -- cgit v1.1 From 91e5b7f4b85cdbc399ccaa1bb1d813e0d829f3d5 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 14:37:01 +0100 Subject: submitcert.py: submit multiple cert chains --- tools/certtools.py | 14 +++++++ tools/submitcert.py | 104 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 78 insertions(+), 40 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 8d64ee4..7b901cf 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -10,6 +10,7 @@ import struct import sys import hashlib import ecdsa +import datetime publickeys = { "https://ct.googleapis.com/pilot/": @@ -204,3 +205,16 @@ def get_leaf_hash(merkle_tree_leaf): leaf_hash.update(merkle_tree_leaf) return leaf_hash.digest() + +def timing_point(timer_dict=None, name=None): + t = datetime.datetime.now() + if timer_dict: + starttime = timer_dict["lasttime"] + stoptime = t + deltatime = stoptime - starttime + timer_dict["deltatimes"].append((name, deltatime.seconds * 1000000 + deltatime.microseconds)) + timer_dict["lasttime"] = t + return None + else: + timer_dict = {"deltatimes":[], "lasttime":t} + return timer_dict diff --git a/tools/submitcert.py b/tools/submitcert.py index 4f1609c..80a3e37 100755 --- a/tools/submitcert.py +++ b/tools/submitcert.py @@ -12,63 +12,87 @@ import struct import hashlib import itertools from certtools import * +import os + +from multiprocessing import Pool baseurl = sys.argv[1] -certfile = sys.argv[2] +certfilepath = sys.argv[2] + +lookup_in_log = False +check_sig = False + +if certfilepath[-1] == "/": + certfiles = [certfilepath + filename for filename in sorted(os.listdir(certfilepath))] +else: + certfiles = [certfilepath] + +def submitcert(certfile): + timing = timing_point() + certs = get_certs_from_file(certfile) + timing_point(timing, "readcerts") + + result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)}) + + timing_point(timing, "addchain") + + try: + if check_sig: + check_sct_signature(baseurl, certs[0], result) + timing_point(timing, "checksig") + except AssertionError, e: + print "ERROR:", e + sys.exit(1) + except ecdsa.keys.BadSignatureError, e: + print "ERROR: bad signature" + sys.exit(1) -lookup_in_log = True + if lookup_in_log: -certs = get_certs_from_file(certfile) + merkle_tree_leaf = pack_mtl(result["timestamp"], certs[0]) -result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)}) + leaf_hash = get_leaf_hash(merkle_tree_leaf) -try: - check_sct_signature(baseurl, certs[0], result) -except AssertionError, e: - print "ERROR:", e - sys.exit(1) -except ecdsa.keys.BadSignatureError, e: - print "ERROR: bad signature" - sys.exit(1) -print "signature check succeeded" + sth = get_sth(baseurl) -if lookup_in_log: + proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"]) - merkle_tree_leaf = pack_mtl(result["timestamp"], certs[0]) + leaf_index = proof["leaf_index"] - leaf_hash = get_leaf_hash(merkle_tree_leaf) + entries = get_entries(baseurl, leaf_index, leaf_index) - sth = get_sth(baseurl) + fetched_entry = entries["entries"][0] - proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"]) + print "does the leaf_input of the fetched entry match what we calculated:", \ + base64.decodestring(fetched_entry["leaf_input"]) == merkle_tree_leaf - leaf_index = proof["leaf_index"] + extra_data = fetched_entry["extra_data"] - entries = get_entries(baseurl, leaf_index, leaf_index) + certchain = decode_certificate_chain(base64.decodestring(extra_data)) - fetched_entry = entries["entries"][0] + submittedcertchain = certs[1:] - print "does the leaf_input of the fetched entry match what we calculated:", \ - base64.decodestring(fetched_entry["leaf_input"]) == merkle_tree_leaf + for (submittedcert, fetchedcert, i) in zip(submittedcertchain, + certchain, itertools.count(1)): + print "cert", i, "in chain is the same:", submittedcert == fetchedcert - extra_data = fetched_entry["extra_data"] + if len(certchain) == len(submittedcertchain) + 1: + last_issuer = get_cert_info(certs[-1])["issuer"] + root_subject = get_cert_info(certchain[-1])["subject"] + print "issuer of last cert in submitted chain and " \ + "subject of last cert in fetched chain is the same:", \ + last_issuer == root_subject + elif len(certchain) == len(submittedcertchain): + print "cert chains are the same length" + else: + print "ERROR: fetched cert chain has length", len(certchain), + print "and submitted chain has length", len(submittedcertchain) - certchain = decode_certificate_chain(base64.decodestring(extra_data)) + timing_point(timing, "lookup") + return timing["deltatimes"] - submittedcertchain = certs[1:] +p = Pool(1) - for (submittedcert, fetchedcert, i) in zip(submittedcertchain, - certchain, itertools.count(1)): - print "cert", i, "in chain is the same:", submittedcert == fetchedcert +for timing in p.imap_unordered(submitcert, certfiles): + print timing - if len(certchain) == len(submittedcertchain) + 1: - last_issuer = get_cert_info(certs[-1])["issuer"] - root_subject = get_cert_info(certchain[-1])["subject"] - print "issuer of last cert in submitted chain and " \ - "subject of last cert in fetched chain is the same:", \ - last_issuer == root_subject - elif len(certchain) == len(submittedcertchain): - print "cert chains are the same length" - else: - print "ERROR: fetched cert chain has length", len(certchain), - print "and submitted chain has length", len(submittedcertchain) -- cgit v1.1 From 44f48b0f96aba0009bd43036eea443f07cec71b9 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 14:37:48 +0100 Subject: Added fetchallcerts.py --- tools/certtools.py | 8 ++++++++ tools/fetchallcerts.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tools/fetchallcerts.py (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 7b901cf..16c2105 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -199,6 +199,14 @@ def pack_mtl(timestamp, leafcert): merkle_tree_leaf = version + leaf_type + timestamped_entry return merkle_tree_leaf +def unpack_mtl(merkle_tree_leaf): + version = merkle_tree_leaf[0:1] + leaf_type = merkle_tree_leaf[1:2] + timestamped_entry = merkle_tree_leaf[2:] + (timestamp, entry_type) = struct.unpack(">QH", timestamped_entry[0:10]) + (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3) + return (leafcert, timestamp) + def get_leaf_hash(merkle_tree_leaf): leaf_hash = hashlib.sha256() leaf_hash.update(struct.pack(">b", 0)) diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py new file mode 100644 index 0000000..801e296 --- /dev/null +++ b/tools/fetchallcerts.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import urllib2 +import urllib +import json +import base64 +import sys +import struct +import hashlib +import itertools +from certtools import * + +def extract_original_entry(entry): + leaf_input = base64.decodestring(entry["leaf_input"]) + (leaf_cert, timestamp) = unpack_mtl(leaf_input) + extra_data = base64.decodestring(entry["extra_data"]) + certchain = decode_certificate_chain(extra_data) + return [leaf_cert] + certchain + +def get_entries_wrapper(baseurl, start, end): + fetched_entries = [] + while len(fetched_entries) < (end - start + 1): + print "fetching from", start + len(fetched_entries) + entries = get_entries(baseurl, start + len(fetched_entries), end)["entries"] + if len(entries) == 0: + break + fetched_entries.extend(entries) + return fetched_entries + +baseurl = sys.argv[1] +destination_directory = sys.argv[2] + +sth = get_sth(baseurl) +tree_size = sth["tree_size"] + +print tree_size + +entries = get_entries_wrapper(baseurl, 0, tree_size) + +print len(entries) + +for entry, i in zip(entries, range(0, len(entries))): + chain = extract_original_entry(entry) + f = open(destination_directory + "/" + ("%06d" % i), "w") + for cert in chain: + print >> f, "-----BEGIN CERTIFICATE-----" + print >> f, base64.encodestring(cert).rstrip() + print >> f, "-----END CERTIFICATE-----" + print >> f, "" -- cgit v1.1 From b9c709204da83be2f315664f9f263c6890b1bc8d Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 16:13:41 +0100 Subject: fetchallcerts.py: calculate root hash --- tools/certtools.py | 24 +++++++++++++++++++++++ tools/fetchallcerts.py | 52 +++++++++++++++++++++++++++++++++++--------------- 2 files changed, 61 insertions(+), 15 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 16c2105..b132caa 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -226,3 +226,27 @@ def timing_point(timer_dict=None, name=None): else: timer_dict = {"deltatimes":[], "lasttime":t} return timer_dict + +def internal_hash(pair): + if len(pair) == 1: + return pair[0] + else: + hash = hashlib.sha256() + hash.update(struct.pack(">b", 1)) + hash.update(pair[0]) + hash.update(pair[1]) + return hash.digest() + +def chunks(l, n): + return [l[i:i+n] for i in range(0, len(l), n)] + +def next_merkle_layer(layer): + return [internal_hash(pair) for pair in chunks(layer, 2)] + +def build_merkle_tree(layer0): + layers = [] + current_layer = layer0 + while len(current_layer) > 1: + current_layer = next_merkle_layer(current_layer) + layers.append(current_layer) + return layers diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py index 801e296..dad5241 100644 --- a/tools/fetchallcerts.py +++ b/tools/fetchallcerts.py @@ -4,6 +4,7 @@ # Copyright (c) 2014, NORDUnet A/S. # See LICENSE for licensing information. +import argparse import urllib2 import urllib import json @@ -14,6 +15,11 @@ import hashlib import itertools from certtools import * +parser = argparse.ArgumentParser(description='') +parser.add_argument('baseurl', help="Base URL for CT server") +parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir') +args = parser.parse_args() + def extract_original_entry(entry): leaf_input = base64.decodestring(entry["leaf_input"]) (leaf_cert, timestamp) = unpack_mtl(leaf_input) @@ -23,7 +29,7 @@ def extract_original_entry(entry): def get_entries_wrapper(baseurl, start, end): fetched_entries = [] - while len(fetched_entries) < (end - start + 1): + while start + len(fetched_entries) < (end + 1): print "fetching from", start + len(fetched_entries) entries = get_entries(baseurl, start + len(fetched_entries), end)["entries"] if len(entries) == 0: @@ -31,23 +37,39 @@ def get_entries_wrapper(baseurl, start, end): fetched_entries.extend(entries) return fetched_entries -baseurl = sys.argv[1] -destination_directory = sys.argv[2] +def print_layer(layer): + for entry in layer: + print base64.b16encode(entry) -sth = get_sth(baseurl) +sth = get_sth(args.baseurl) tree_size = sth["tree_size"] +root_hash = base64.decodestring(sth["sha256_root_hash"]) + +print "tree size", tree_size +print "root hash", base64.b16encode(root_hash) + +entries = get_entries_wrapper(args.baseurl, 0, tree_size - 1) + +print "fetched", len(entries), "entries" + +layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries] + +tree = build_merkle_tree(layer0) -print tree_size +calculated_root_hash = tree[-1][0] -entries = get_entries_wrapper(baseurl, 0, tree_size) +print "calculated root hash", base64.b16encode(calculated_root_hash) -print len(entries) +if calculated_root_hash != root_hash: + print "fetched root hash and calculated root hash different, aborting" + sys.exit(1) -for entry, i in zip(entries, range(0, len(entries))): - chain = extract_original_entry(entry) - f = open(destination_directory + "/" + ("%06d" % i), "w") - for cert in chain: - print >> f, "-----BEGIN CERTIFICATE-----" - print >> f, base64.encodestring(cert).rstrip() - print >> f, "-----END CERTIFICATE-----" - print >> f, "" +if args.store: + for entry, i in zip(entries, range(0, len(entries))): + chain = extract_original_entry(entry) + f = open(args.store + "/" + ("%06d" % i), "w") + for cert in chain: + print >> f, "-----BEGIN CERTIFICATE-----" + print >> f, base64.encodestring(cert).rstrip() + print >> f, "-----END CERTIFICATE-----" + print >> f, "" -- cgit v1.1 From 89b8a59da38daf465a934f039cc7cf3efaf1468e Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 16:56:03 +0100 Subject: merge.py: send whole sth in sendsth call --- tools/certtools.py | 20 ++++++++++++++++++++ tools/merge.py | 27 ++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index b132caa..0ce8885 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -143,6 +143,11 @@ def decode_signature(signature): assert rest == "" return (hash_alg, signature_alg, unpacked_signature) +def encode_signature(hash_alg, signature_alg, unpacked_signature): + signature = struct.pack(">bb", hash_alg, signature_alg) + signature += tls_array(unpacked_signature, 2) + return signature + def check_signature(baseurl, signature, data): publickey = base64.decodestring(publickeys[baseurl]) (hash_alg, signature_alg, unpacked_signature) = decode_signature(signature) @@ -155,6 +160,12 @@ def check_signature(baseurl, signature, data): vk.verify(unpacked_signature, data, hashfunc=hashlib.sha256, sigdecode=ecdsa.util.sigdecode_der) +def create_signature(privatekey, data): + sk = ecdsa.SigningKey.from_der(privatekey) + unpacked_signature = sk.sign(data, hashfunc=hashlib.sha256, + sigencode=ecdsa.util.sigencode_der) + return encode_signature(4, 3, unpacked_signature) + def check_sth_signature(baseurl, sth): signature = base64.decodestring(sth["tree_head_signature"]) @@ -167,6 +178,15 @@ def check_sth_signature(baseurl, sth): check_signature(baseurl, signature, tree_head) +def create_sth_signature(tree_size, timestamp, root_hash, privatekey): + version = struct.pack(">b", 0) + signature_type = struct.pack(">b", 1) + timestamp_packed = struct.pack(">Q", timestamp) + tree_size_packed = struct.pack(">Q", tree_size) + tree_head = version + signature_type + timestamp_packed + tree_size_packed + root_hash + + return create_signature(privatekey, tree_head) + def check_sct_signature(baseurl, leafcert, sct): publickey = base64.decodestring(publickeys[baseurl]) calculated_logid = hashlib.sha256(publickey).digest() diff --git a/tools/merge.py b/tools/merge.py index 3aeecdf..7c6e357 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -9,7 +9,10 @@ import base64 import urllib import urllib2 import sys +import time +from certtools import build_merkle_tree, create_sth_signature, check_sth_signature +ctbaseurl = "https://127.0.0.1:8080/" frontendnodes = ["https://127.0.0.1:8082/"] storagenodes = ["https://127.0.0.1:8081/"] @@ -142,6 +145,28 @@ for new_entry in new_entries: added_entries += 1 print "added", added_entries, "entries" +tree = build_merkle_tree(logorder) +tree_size = len(logorder) +root_hash = tree[-1][0] +timestamp = int(time.time() * 1000) +privatekey = base64.decodestring( + "MHcCAQEEIMM/FjZ4FSzfENTTwGpTve6CP+IVr" + "Y7p8OKV634uJI/foAoGCCqGSM49AwEHoUQDQg" + "AE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9PMS5l" + "qoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJ" + "GolozZYmNHE2kQ==") + +tree_head_signature = create_sth_signature(tree_size, timestamp, + root_hash, privatekey) + +sth = {"tree_size": tree_size, "timestamp": timestamp, + "sha256_root_hash": base64.b64encode(root_hash), + "tree_head_signature": base64.b64encode(tree_head_signature)} + +check_sth_signature(ctbaseurl, sth) + +print "root hash", base64.b16encode(root_hash) + for frontendnode in frontendnodes: print "distributing for node", frontendnode curpos = get_curpos(frontendnode) @@ -154,4 +179,4 @@ for frontendnode in frontendnodes: for missingentry in missingentries: hash = base64.b64decode(missingentry) sendentry(frontendnode, read_chain(hash), hash) - sendsth(frontendnode, {"tree_size": len(logorder)}) + sendsth(frontendnode, sth) -- cgit v1.1 From 8de65ab10cbdf6ea5509f321f025686e6ab0ecaf Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 28 Oct 2014 16:03:11 +0100 Subject: certtools.py: fix bug in build_merkle_tree --- tools/certtools.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 0ce8885..cbb4ff7 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -264,8 +264,11 @@ def next_merkle_layer(layer): return [internal_hash(pair) for pair in chunks(layer, 2)] def build_merkle_tree(layer0): + if len(layer0) == 0: + return [[hashlib.sha256().digest()]] layers = [] current_layer = layer0 + layers.append(current_layer) while len(current_layer) > 1: current_layer = next_merkle_layer(current_layer) layers.append(current_layer) -- cgit v1.1 From ce6fac1d34f71732e56d5b4e3f4819d6fe282957 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 28 Oct 2014 16:04:06 +0100 Subject: Check return value from merge.py --- tools/testcase1.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/tools/testcase1.py b/tools/testcase1.py index dd3597e..639cd69 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -122,11 +122,15 @@ def get_and_check_entry(timestamp, chain, leaf_index): print_and_check_tree_size(0) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) + testgroup("cert1") result1 = do_add_chain(cc1) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(1) @@ -134,7 +138,8 @@ result2 = do_add_chain(cc1) assert_equal(result2["timestamp"], result1["timestamp"], "timestamp") -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(1) @@ -147,7 +152,8 @@ testgroup("cert2") result3 = do_add_chain(cc2) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(2) @@ -158,7 +164,8 @@ testgroup("cert3") result4 = do_add_chain(cc3) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(3) @@ -170,7 +177,8 @@ testgroup("cert4") result5 = do_add_chain(cc4) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(4) @@ -183,7 +191,8 @@ testgroup("cert5") result6 = do_add_chain(cc5) -subprocess.call(["./merge.py"]) +mergeresult = subprocess.call(["./merge.py"]) +assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(5) -- cgit v1.1