extracting monitor from auditor

8 files changed, 1429 insertions, 41 deletions

diff --git a/monitor/certkeys.py b/monitor/certkeys.py
new file mode 100644
index 0000000..f7c83b1
--- /dev/null
+++ b/monitor/certkeys.py
@@ -0,0 +1,43 @@
+
+publickeys = {
+    "https://ct.googleapis.com/pilot/":
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTD"
+    "M0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==",
+
+    "https://plausible.ct.nordu.net/":
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUB"
+	"c8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ==",
+
+    "https://ct1.digicert-ct.com/log/":
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCF"
+	"RkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A==",
+
+    "https://ct.izenpe.com/":
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJ2Q5DC3cUBj4IQCiDu0s6j51up+T"
+	"ZAkAEcQRF6tczw90rLWXkJMAW7jr9yc92bIKgV8vDXU4lDeZHvYHduDuvg==",
+    
+    "https://log.certly.io/":
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECyPLhWKYYUgEc+tUXfPQB4wtGS2M"
+	"NvXrjwFCCnyYJifBtd2Sk7Cu+Js9DNhMTh35FftHaHu6ZrclnNBKwmbbSA==", 
+
+    "https://ct.googleapis.com/aviator/":
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1/TMabLkDpCjiupacAlP7xNi0I1J"
+	"YP8bQFAHDG1xhtolSY1l4QgNRzRrvSe8liE+NPWHdjGxfx3JhTsN9x8/6Q==",
+
+    "https://ct.googleapis.com/rocketeer/":
+   	"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1"
+	"aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg==",
+
+    "https://ct.ws.symantec.com/":
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEluqsHEYMG1XcDfy1lCdGV0JwOmkY"
+	"4r87xNuroPS2bMBTP01CEDPwWJePa75y9CrsHEKqAy8afig1dpkIPSEUhg==",
+    
+    "https://ctlog.api.venafi.com/":
+    "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAolpIHxdSlTXLo1s6H1OC"
+	"dpSj/4DyHDc8wLG9wVmLqy1lk9fz4ATVmm+/1iN2Nk8jmctUKK2MFUtlWXZBSpym"
+	"97M7frGlSaQXUWyA3CqQUEuIJOmlEjKTBEiQAvpfDjCHjlV2Be4qTM6jamkJbiWt"
+	"gnYPhJL6ONaGTiSPm7Byy57iaz/hbckldSOIoRhYBiMzeNoA0DiRZ9KmfSeXZ1rB"
+	"8y8X5urSW+iBzf2SaOfzBvDpcoTuAaWx2DPazoOl28fP1hZ+kHUYvxbcMjttjauC"
+	"Fx+JII0dmuZNIwjfeG/GBb9frpSX219k1O4Wi6OEbHEr8at/XQ0y7gTikOxBn/s5"
+	"wQIDAQAB",
+}
diff --git a/monitor/josef_lib.py b/monitor/josef_lib.py
new file mode 100644
index 0000000..3c52761
--- /dev/null
+++ b/monitor/josef_lib.py
@@ -0,0 +1,779 @@
+# Copyright (c) 2014, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+import subprocess
+import json
+import base64
+import urllib
+import urllib2
+import ssl
+import urlparse
+import struct
+import sys
+import hashlib
+import ecdsa
+import datetime
+import cStringIO
+import zipfile
+import shutil
+from certkeys import publickeys
+
+from Crypto.Hash import SHA256
+import Crypto.PublicKey.RSA as RSA
+from Crypto.Signature import PKCS1_v1_5
+
+def get_cert_info(s):
+    p = subprocess.Popen(
+        ["openssl", "x509", "-noout", "-subject", "-issuer", "-inform", "der"],
+        stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    parsed = p.communicate(s)
+    if parsed[1]:
+        print "ERROR:", parsed[1]
+        sys.exit(1)
+    result = {}
+    for line in parsed[0].split("\n"):
+        (key, sep, value) = line.partition("=")
+        if sep == "=":
+            result[key] = value
+    return result
+
+def my_get_cert_info(s):
+    p = subprocess.Popen(
+        ["openssl", "x509", "-fingerprint", "-text", "-noout", "-inform", "der"],
+        stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    parsed = p.communicate(s)
+    if parsed[1]:
+        print "ERROR:", parsed[1]
+        sys.exit(1)
+    result = {}
+    prev = ""
+    for line in parsed[0].split("\n"):
+        if "Subject:" in line:
+            result["subject"] = line.split("Subject: ")[1]
+        if "Issuer:" in line:
+            result["issuer"] = line.split("Issuer: ")[1]
+        if "Subject Alternative Name" in prev:
+            result["SAN"] = line.lstrip()
+        if "Not After" in line:
+            result["not_after"] = line.split(": ")[1]
+        if "Not Before" in line:
+            result["not_before"] = line.split(": ")[1]
+        prev = line
+    return result
+
+def get_pemlike(filename, marker):
+    return get_pemlike_from_file(open(filename), marker)
+
+def get_pemlike_from_file(f, marker):
+    entries = []
+    entry = ""
+    inentry = False
+
+    for line in f:
+        line = line.strip()
+        if line == "-----BEGIN " + marker + "-----":
+            entry = ""
+            inentry = True
+        elif line == "-----END " + marker + "-----":
+            entries.append(base64.decodestring(entry))
+            inentry = False
+        elif inentry:
+            entry += line
+    return entries
+
+def get_certs_from_file(certfile):
+    return get_pemlike(certfile, "CERTIFICATE")
+
+def get_certs_from_string(s):
+    f = cStringIO.StringIO(s)
+    return get_pemlike_from_file(f, "CERTIFICATE")
+
+def get_precerts_from_string(s):
+    f = cStringIO.StringIO(s)
+    return get_pemlike_from_file(f, "PRECERTIFICATE")
+
+def get_eckey_from_file(keyfile):
+    keys = get_pemlike(keyfile, "EC PRIVATE KEY")
+    assert len(keys) == 1
+    return keys[0]
+
+def get_public_key_from_file(keyfile):
+    keys = get_pemlike(keyfile, "PUBLIC KEY")
+    assert len(keys) == 1
+    return keys[0]
+
+def get_root_cert(issuer):
+    accepted_certs = \
+        json.loads(open("googlelog-accepted-certs.txt").read())["certificates"]
+
+    root_cert = None
+
+    for accepted_cert in accepted_certs:
+        subject = get_cert_info(base64.decodestring(accepted_cert))["subject"]
+        if subject == issuer:
+            root_cert = base64.decodestring(accepted_cert)
+
+    return root_cert
+
+class sslparameters:
+    sslcontext = None
+
+def create_ssl_context(cafile=None):
+    try:
+        sslparameters.sslcontext = ssl.create_default_context(cafile=cafile)
+    except AttributeError:
+        sslparameters.sslcontext = None
+
+def get_opener():
+    try:
+        opener = urllib2.build_opener(urllib2.HTTPSHandler(context=sslparameters.sslcontext))
+    except TypeError:
+        opener = urllib2.build_opener(urllib2.HTTPSHandler())
+    return opener
+
+def urlopen(url, data=None):
+    return get_opener().open(url, data)
+
+def pyopenssl_https_get(url):
+    """
+    HTTPS GET-function to use when running old Python < 2.7
+    """
+    from OpenSSL import SSL
+    import socket
+
+    # TLSv1 is the best we can get on Python 2.6
+    context = SSL.Context(SSL.TLSv1_METHOD)
+    sock = SSL.Connection(context, socket.socket(socket.AF_INET, socket.SOCK_STREAM))
+
+    url_without_scheme = url.split('https://')[-1]
+    host = url_without_scheme.split('/')[0]
+    path = url_without_scheme.split('/', 1)[1]
+    http_get_request = ("GET /{path} HTTP/1.1\r\n"
+                        "Host: {host}\r\n"
+                        "\r\n"
+                        ).format(path=path, host=host)
+
+    sock.connect((host, 443))
+    sock.write(http_get_request)
+    response = sock.recv(1024)
+    response_lines = response.rsplit('\n')
+
+    # We are only interested in the actual response,
+    # without headers, contained in the last line.
+    return response_lines[len(response_lines) - 1]
+
+def get_sth(baseurl):
+    result = urlopen(baseurl + "ct/v1/get-sth").read()
+    return json.loads(result)
+
+def get_proof_by_hash(baseurl, hash, tree_size):
+    try:
+        params = urllib.urlencode({"hash":base64.b64encode(hash),
+                                   "tree_size":tree_size})
+        result = \
+          urlopen(baseurl + "ct/v1/get-proof-by-hash?" + params).read()
+        return json.loads(result)
+    except urllib2.HTTPError, e:
+        print "ERROR:", e.read()
+        sys.exit(1)
+
+def get_consistency_proof(baseurl, tree_size1, tree_size2):
+    try:
+        params = urllib.urlencode({"first":tree_size1,
+                                   "second":tree_size2})
+        result = \
+          urlopen(baseurl + "ct/v1/get-sth-consistency?" + params).read()
+        return json.loads(result)["consistency"]
+    except urllib2.HTTPError, e:
+        print "ERROR:", e.read()
+        sys.exit(1)
+
+def tls_array(data, length_len):
+    length_bytes = struct.pack(">Q", len(data))[-length_len:]
+    return length_bytes + data
+
+def unpack_tls_array(packed_data, length_len):
+    padded_length = ["\x00"] * 8
+    padded_length[-length_len:] = packed_data[:length_len]
+    (length,) = struct.unpack(">Q", "".join(padded_length))
+    unpacked_data = packed_data[length_len:length_len+length]
+    assert len(unpacked_data) == length, \
+      "data is only %d bytes long, but length is %d bytes" % \
+      (len(unpacked_data), length)
+    rest_data = packed_data[length_len+length:]
+    return (unpacked_data, rest_data)
+
+def add_chain(baseurl, submission):
+    try:
+        result = urlopen(baseurl + "ct/v1/add-chain", json.dumps(submission)).read()
+        return json.loads(result)
+    except urllib2.HTTPError, e:
+        print "ERROR", e.code,":", e.read()
+        if e.code == 400:
+            return None
+        sys.exit(1)
+    except ValueError, e:
+        print "==== FAILED REQUEST ===="
+        print submission
+        print "======= RESPONSE ======="
+        print result
+        print "========================"
+        raise e
+
+def add_prechain(baseurl, submission):
+    try:
+        result = urlopen(baseurl + "ct/v1/add-pre-chain",
+            json.dumps(submission)).read()
+        return json.loads(result)
+    except urllib2.HTTPError, e:
+        print "ERROR", e.code,":", e.read()
+        if e.code == 400:
+            return None
+        sys.exit(1)
+    except ValueError, e:
+        print "==== FAILED REQUEST ===="
+        print submission
+        print "======= RESPONSE ======="
+        print result
+        print "========================"
+        raise e
+
+def get_entries(baseurl, start, end):
+    params = urllib.urlencode({"start":start, "end":end})
+    try:
+        result = urlopen(baseurl + "ct/v1/get-entries?" + params).read()
+        return json.loads(result)
+    except urllib2.HTTPError, e:
+        print "ERROR:", e.read()
+        sys.exit(1)
+
+def extract_precertificate(precert_chain_entry):
+    (precert, certchain) = unpack_tls_array(precert_chain_entry, 3)
+    return (precert, certchain)
+
+def decode_certificate_chain(packed_certchain):
+    (unpacked_certchain, rest) = unpack_tls_array(packed_certchain, 3)
+    assert len(rest) == 0
+    certs = []
+    while len(unpacked_certchain):
+        (cert, rest) = unpack_tls_array(unpacked_certchain, 3)
+        certs.append(cert)
+        unpacked_certchain = rest
+    return certs
+
+def decode_signature(signature):
+    (hash_alg, signature_alg) = struct.unpack(">bb", signature[0:2])
+    (unpacked_signature, rest) = unpack_tls_array(signature[2:], 2)
+    assert rest == ""
+    return (hash_alg, signature_alg, unpacked_signature)
+
+def encode_signature(hash_alg, signature_alg, unpacked_signature):
+    signature = struct.pack(">bb", hash_alg, signature_alg)
+    signature += tls_array(unpacked_signature, 2)
+    return signature
+
+def check_signature(baseurl, signature, data, publickey=None):
+    if publickey == None:
+        if baseurl in publickeys:
+            publickey = base64.decodestring(publickeys[baseurl])
+        else:
+            print >>sys.stderr, "Public key for", baseurl, \
+                "not found, specify key file with --publickey"
+            sys.exit(1)
+    (hash_alg, signature_alg, unpacked_signature) = decode_signature(signature)
+    assert hash_alg == 4, \
+        "hash_alg is %d, expected 4" % (hash_alg,) # sha256
+    assert (signature_alg == 3 or signature_alg == 1), \
+        "signature_alg is %d, expected 1 or 3" % (signature_alg,) # ecdsa
+
+    if signature_alg == 3:
+        vk = ecdsa.VerifyingKey.from_der(publickey)
+        vk.verify(unpacked_signature, data, hashfunc=hashlib.sha256,
+              sigdecode=ecdsa.util.sigdecode_der)
+    else:
+        h = SHA256.new(data)
+        rsa_key = RSA.importKey(publickey)
+        verifier = PKCS1_v1_5.new(rsa_key)
+        assert verifier.verify(h, unpacked_signature), \
+            "could not verify RSA signature"
+
+def parse_auth_header(authheader):
+    splittedheader = authheader.split(";")
+    (signature, rawoptions) = (splittedheader[0], splittedheader[1:])
+    options = dict([(e.partition("=")[0], e.partition("=")[2]) for e in rawoptions])
+    return (base64.b64decode(signature), options)
+
+def check_auth_header(authheader, expected_key, publickeydir, data, path):
+    if expected_key == None:
+        return True
+    (signature, options) = parse_auth_header(authheader)
+    keyname = options.get("key")
+    if keyname != expected_key:
+        raise Exception("Response claimed to come from %s, expected %s" % (keyname, expected_key))
+    publickey = get_public_key_from_file(publickeydir + "/" + keyname + ".pem")
+    vk = ecdsa.VerifyingKey.from_der(publickey)
+    vk.verify(signature, "%s\0%s\0%s" % ("REPLY", path, data), hashfunc=hashlib.sha256,
+              sigdecode=ecdsa.util.sigdecode_der)
+    return True
+
+def http_request(url, data=None, key=None, verifynode=None, publickeydir="."):
+    opener = get_opener()
+
+    (keyname, keyfile) = key
+    privatekey = get_eckey_from_file(keyfile)
+    sk = ecdsa.SigningKey.from_der(privatekey)
+    parsed_url = urlparse.urlparse(url)
+    if data == None:
+        data_to_sign = parsed_url.query
+        method = "GET"
+    else:
+        data_to_sign = data
+        method = "POST"
+    signature = sk.sign("%s\0%s\0%s" % (method, parsed_url.path, data_to_sign), hashfunc=hashlib.sha256,
+                        sigencode=ecdsa.util.sigencode_der)
+    opener.addheaders = [('X-Catlfish-Auth', base64.b64encode(signature) + ";key=" + keyname)]
+    result = opener.open(url, data)
+    authheader = result.info().get('X-Catlfish-Auth')
+    data = result.read()
+    check_auth_header(authheader, verifynode, publickeydir, data, parsed_url.path)
+    return data
+
+def get_signature(baseurl, data, key=None):
+    try:
+        params = json.dumps({"plop_version":1, "data": base64.b64encode(data)})
+        result = http_request(baseurl + "plop/v1/signing/sth", params, key=key)
+        parsed_result = json.loads(result)
+        return base64.b64decode(parsed_result.get(u"result"))
+    except urllib2.HTTPError, e:
+        print "ERROR: get_signature", e.read()
+        raise e
+
+def create_signature(baseurl, data, key=None):
+    unpacked_signature = get_signature(baseurl, data, key)
+    return encode_signature(4, 3, unpacked_signature)
+
+def check_sth_signature(baseurl, sth, publickey=None):
+    signature = base64.decodestring(sth["tree_head_signature"])
+
+    version = struct.pack(">b", 0)
+    signature_type = struct.pack(">b", 1)
+    timestamp = struct.pack(">Q", sth["timestamp"])
+    tree_size = struct.pack(">Q", sth["tree_size"])
+    hash = base64.decodestring(sth["sha256_root_hash"])
+    tree_head = version + signature_type + timestamp + tree_size + hash
+
+    check_signature(baseurl, signature, tree_head, publickey=publickey)
+
+def create_sth_signature(tree_size, timestamp, root_hash, baseurl, key=None):
+    version = struct.pack(">b", 0)
+    signature_type = struct.pack(">b", 1)
+    timestamp_packed = struct.pack(">Q", timestamp)
+    tree_size_packed = struct.pack(">Q", tree_size)
+    tree_head = version + signature_type + timestamp_packed + tree_size_packed + root_hash
+
+    return create_signature(baseurl, tree_head, key=key)
+
+def check_sct_signature(baseurl, signed_entry, sct, precert=False, publickey=None):
+    if publickey == None:
+        publickey = base64.decodestring(publickeys[baseurl])
+    calculated_logid = hashlib.sha256(publickey).digest()
+    received_logid = base64.decodestring(sct["id"])
+    assert calculated_logid == received_logid, \
+        "log id is incorrect:\n  should be %s\n        got %s" % \
+        (calculated_logid.encode("hex_codec"),
+         received_logid.encode("hex_codec"))
+
+    signature = base64.decodestring(sct["signature"])
+
+    version = struct.pack(">b", sct["sct_version"])
+    signature_type = struct.pack(">b", 0)
+    timestamp = struct.pack(">Q", sct["timestamp"])
+    if precert:
+        entry_type = struct.pack(">H", 1)
+    else:
+        entry_type = struct.pack(">H", 0)
+    signed_struct = version + signature_type + timestamp + \
+      entry_type + signed_entry + \
+      tls_array(base64.decodestring(sct["extensions"]), 2)
+
+    check_signature(baseurl, signature, signed_struct, publickey=publickey)
+
+def pack_mtl(timestamp, leafcert):
+    entry_type = struct.pack(">H", 0)
+    extensions = ""
+
+    timestamped_entry = struct.pack(">Q", timestamp) + entry_type + \
+      tls_array(leafcert, 3) + tls_array(extensions, 2)
+    version = struct.pack(">b", 0)
+    leaf_type = struct.pack(">b", 0)
+    merkle_tree_leaf = version + leaf_type + timestamped_entry
+    return merkle_tree_leaf
+
+def pack_mtl_precert(timestamp, cleanedcert, issuer_key_hash):
+    entry_type = struct.pack(">H", 1)
+    extensions = ""
+
+    timestamped_entry = struct.pack(">Q", timestamp) + entry_type + \
+      pack_precert(cleanedcert, issuer_key_hash) + tls_array(extensions, 2)
+    version = struct.pack(">b", 0)
+    leaf_type = struct.pack(">b", 0)
+    merkle_tree_leaf = version + leaf_type + timestamped_entry
+    return merkle_tree_leaf
+
+def pack_precert(cleanedcert, issuer_key_hash):
+    assert len(issuer_key_hash) == 32
+
+    return issuer_key_hash + tls_array(cleanedcert, 3)
+
+def pack_cert(cert):
+    return tls_array(cert, 3)
+
+def unpack_mtl(merkle_tree_leaf):
+    version = merkle_tree_leaf[0:1]
+    leaf_type = merkle_tree_leaf[1:2]
+    timestamped_entry = merkle_tree_leaf[2:]
+    (timestamp, entry_type) = struct.unpack(">QH", timestamped_entry[0:10])
+    if entry_type == 0:
+        issuer_key_hash = None
+        (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3)
+    elif entry_type == 1:
+        issuer_key_hash = timestamped_entry[10:42]
+        (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[42:], 3)
+    return (leafcert, timestamp, issuer_key_hash)
+
+def get_leaf_hash(merkle_tree_leaf):
+    leaf_hash = hashlib.sha256()
+    leaf_hash.update(struct.pack(">b", 0))
+    leaf_hash.update(merkle_tree_leaf)
+
+    return leaf_hash.digest()
+
+def timing_point(timer_dict=None, name=None):
+    t = datetime.datetime.now()
+    if timer_dict:
+        starttime = timer_dict["lasttime"]
+        stoptime = t
+        deltatime = stoptime - starttime
+        timer_dict["deltatimes"].append((name, deltatime.seconds * 1000000 + deltatime.microseconds))
+        timer_dict["lasttime"] = t
+        return None
+    else:
+        timer_dict = {"deltatimes":[], "lasttime":t}
+        return timer_dict
+
+def internal_hash(pair):
+    if len(pair) == 1:
+        return pair[0]
+    else:
+        hash = hashlib.sha256()
+        hash.update(struct.pack(">b", 1))
+        hash.update(pair[0])
+        hash.update(pair[1])
+        digest = hash.digest()
+        return digest
+
+def chunks(l, n):
+    return [l[i:i+n] for i in range(0, len(l), n)]
+
+def next_merkle_layer(layer):
+    return [internal_hash(pair) for pair in chunks(layer, 2)]
+
+def build_merkle_tree(layer0):
+    if len(layer0) == 0:
+        return [[hashlib.sha256().digest()]]
+    layers = []
+    current_layer = layer0
+    layers.append(current_layer)
+    while len(current_layer) > 1:
+        current_layer = next_merkle_layer(current_layer)
+        layers.append(current_layer)
+    return layers
+
+def print_inclusion_proof(proof):
+    audit_path = proof[u'audit_path']
+    n = proof[u'leaf_index']
+    level = 0
+    for s in audit_path:
+        entry = base64.b16encode(base64.b64decode(s))
+        n ^= 1
+        print level, n, entry
+        n >>= 1
+        level += 1
+
+def get_one_cert(store, i):
+    filename = i / 10000
+    zf = zipfile.ZipFile("%s/%04d.zip" % (store, i / 10000))
+    cert = zf.read("%08d" % i)
+    zf.close()
+    return cert
+
+def get_hash_from_certfile(cert):
+    for line in cert.split("\n"):
+        if line.startswith("-----"):
+            return None
+        if line.startswith("Leafhash: "):
+            return base64.b16decode(line[len("Leafhash: "):])
+    return None
+
+def get_timestamp_from_certfile(cert):
+    for line in cert.split("\n"):
+        if line.startswith("-----"):
+            return None
+        if line.startswith("Timestamp: "):
+            return int(line[len("Timestamp: "):])
+    return None
+
+def get_proof(store, tree_size, n):
+    hash = get_hash_from_certfile(get_one_cert(store, n))
+    return get_proof_by_hash(args.baseurl, hash, tree_size)
+
+def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf):
+    for i in range(firstleaf, lastleaf + 1):
+        try:
+            yield zipfiles[i / 10000].read("%08d" % i)
+        except KeyError:
+            return
+
+def get_merkle_hash_64k(store, blocknumber, write_to_cache=False, treesize=None):
+    firstleaf = blocknumber * 65536
+    lastleaf = firstleaf + 65535
+    if treesize != None:
+        assert firstleaf < treesize
+        usecache = lastleaf < treesize
+        lastleaf = min(lastleaf, treesize - 1)
+    else:
+        usecache = True
+
+    hashfilename = "%s/%04x.64khash" % (store, blocknumber)
+    if usecache:
+        try:
+            hash = base64.b16decode(open(hashfilename).read())
+            assert len(hash) == 32
+            return ("hash", hash)
+        except IOError:
+            pass
+    firstfile = firstleaf / 10000
+    lastfile = lastleaf / 10000
+    zipfiles = {}
+    for i in range(firstfile, lastfile + 1):
+        try:
+            zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i))
+        except IOError:
+            break
+    certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf)
+    layer0 = [get_hash_from_certfile(cert) for cert in certs]
+    tree = build_merkle_tree(layer0)
+    calculated_hash = tree[-1][0]
+    for zf in zipfiles.values():
+        zf.close()
+    if len(layer0) != lastleaf - firstleaf + 1:
+        return ("incomplete", (len(layer0), calculated_hash))
+    if write_to_cache:
+        f = open(hashfilename, "w")
+        f.write(base64.b16encode(calculated_hash))
+        f.close()
+    return ("hash", calculated_hash)
+
+def get_tree_head(store, treesize):
+    merkle_64klayer = []
+
+    for blocknumber in range(0, (treesize / 65536) + 1):
+        (resulttype, result) = get_merkle_hash_64k(store, blocknumber, treesize=treesize)
+        if resulttype == "incomplete":
+            print >>sys.stderr, "Couldn't read until tree size", treesize
+            (incompletelength, hash) = result
+            print >>sys.stderr, "Stopped at", blocknumber * 65536 + incompletelength
+            sys.exit(1)
+        assert resulttype == "hash"
+        hash = result
+        merkle_64klayer.append(hash)
+        #print >>sys.stderr, print blocknumber * 65536,
+        sys.stdout.flush()
+    tree = build_merkle_tree(merkle_64klayer)
+    calculated_root_hash = tree[-1][0]
+    return calculated_root_hash
+
+def get_intermediate_hash(store, treesize, level, index):
+    if level >= 16:
+        merkle_64klayer = []
+
+        levelsize = (2**(level-16))
+
+        for blocknumber in range(index * levelsize, (index + 1) * levelsize):
+            if blocknumber * (2 ** 16) >= treesize:
+                break
+            #print "looking at block", blocknumber
+            (resulttype, result) = get_merkle_hash_64k(store, blocknumber, treesize=treesize)
+            if resulttype == "incomplete":
+                print >>sys.stderr, "Couldn't read until tree size", treesize
+                (incompletelength, hash) = result
+                print >>sys.stderr, "Stopped at", blocknumber * 65536 + incompletelength
+                sys.exit(1)
+            assert resulttype == "hash"
+            hash = result
+            #print "block hash", base64.b16encode(hash)
+            merkle_64klayer.append(hash)
+            #print >>sys.stderr, print blocknumber * 65536,
+            sys.stdout.flush()
+        tree = build_merkle_tree(merkle_64klayer)
+        return tree[-1][0]
+    else:
+        levelsize = 2 ** level
+        firstleaf = index * levelsize
+        lastleaf = firstleaf + levelsize - 1
+        #print "firstleaf", firstleaf
+        #print "lastleaf", lastleaf
+        assert firstleaf < treesize
+        lastleaf = min(lastleaf, treesize - 1)
+        #print "modified lastleaf", lastleaf
+        firstfile = firstleaf / 10000
+        lastfile = lastleaf / 10000
+        #print "files", firstfile, lastfile
+        zipfiles = {}
+        for i in range(firstfile, lastfile + 1):
+            try:
+                zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i))
+            except IOError:
+                break
+        certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf)
+        layer0 = [get_hash_from_certfile(cert) for cert in certs]
+        #print "layer0", repr(layer0)
+        tree = build_merkle_tree(layer0)
+        calculated_hash = tree[-1][0]
+        for zf in zipfiles.values():
+            zf.close()
+        assert len(layer0) == lastleaf - firstleaf + 1
+        return calculated_hash
+
+def bits(n):
+    p = 0
+    while n > 0:
+        n >>= 1
+        p += 1
+    return p
+
+def merkle_height(n):
+    if n == 0:
+        return 1
+    return bits(n - 1)
+
+def node_above((pathp, pathl), levels=1):
+    return (pathp >> levels, pathl + levels)
+
+def node_even((pathp, pathl)):
+    return pathp & 1 == 0
+
+def node_odd((pathp, pathl)):
+    return pathp & 1 == 1
+
+def node_lower((path1p, path1l), (path2p, path2l)):
+    return path1l < path2l
+
+def node_higher((path1p, path1l), (path2p, path2l)):
+    return path1l > path2l
+
+def node_level((path1p, path1l)):
+    return path1l
+
+def node_outside((path1p, path1l), (path2p, path2l)):
+    assert path1l == path2l
+    return path1p > path2p
+
+def combine_two_hashes((path1, hash1), (path2, hash2), treesize):
+    assert not node_higher(path1, path2)
+    edge_node = (treesize - 1, 0)
+
+    if node_lower(path1, path2):
+        assert path1 == node_above(edge_node, levels=node_level(path1))
+        while node_even(path1):
+            path1 = node_above(path1)
+
+    assert node_above(path1) == node_above(path2)
+    assert (node_even(path1) and node_odd(path2)) or (node_odd(path1) and node_even(path2))
+
+    if node_outside(path2, node_above(edge_node, levels=node_level(path2))):
+        return (node_above(path1), hash1)
+
+    if node_even(path1):
+        newhash = internal_hash((hash1, hash2))
+    else:
+        newhash = internal_hash((hash2, hash1))
+
+    return (node_above(path1), newhash)
+
+def path_as_string(pos, level, treesize):
+    height = merkle_height(treesize)
+    path = "{0:0{width}b}".format(pos, width=height - level)
+    if height == level:
+        return ""
+    return path
+
+def nodes_for_subtree(subtreesize, treesize):
+    height = merkle_height(treesize)
+    nodes = []
+    level = 0
+    pos = subtreesize
+    while pos > 0 and pos & 1 == 0:
+        pos >>= 1
+        level += 1
+    if pos & 1:
+        nodes.append((pos ^ 1, level))
+    #print pos, level
+    while level < height:
+        pos_level0 = pos * (2 ** level)
+        #print pos, level
+        if pos_level0 < treesize:
+            nodes.append((pos, level))
+        pos >>= 1
+        pos ^= 1
+        level += 1
+    return nodes
+
+def nodes_for_index(pos, treesize):
+    height = merkle_height(treesize)
+    nodes = []
+    level = 0
+    pos ^= 1
+    while level < height:
+        pos_level0 = pos * (2 ** level)
+        if pos_level0 < treesize:
+            nodes.append((pos, level))
+        pos >>= 1
+        pos ^= 1
+        level += 1
+    return nodes
+
+def verify_consistency_proof(consistency_proof, first, second, oldhash_input):
+    if 2 ** bits(first - 1) == first:
+        consistency_proof = [oldhash_input] + consistency_proof
+    chain = zip(nodes_for_subtree(first, second), consistency_proof)
+    assert len(nodes_for_subtree(first, second)) == len(consistency_proof)
+    (_, hash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, second), chain)
+    (_, oldhash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, first), chain)
+    return (oldhash, hash)
+
+def verify_inclusion_proof(inclusion_proof, index, treesize, leafhash):
+    chain = zip([(index, 0)] + nodes_for_index(index, treesize), [leafhash] + inclusion_proof)
+    assert len(nodes_for_index(index, treesize)) == len(inclusion_proof)
+    (_, hash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, treesize), chain)
+    return hash
+
+def extract_original_entry(entry):
+    leaf_input =  base64.decodestring(entry["leaf_input"])
+    (leaf_cert, timestamp, issuer_key_hash) = unpack_mtl(leaf_input)
+    extra_data = base64.decodestring(entry["extra_data"])
+    if issuer_key_hash != None:
+        (precert, extra_data) = extract_precertificate(extra_data)
+        leaf_cert = precert
+    certchain = decode_certificate_chain(extra_data)
+    return ([leaf_cert] + certchain, timestamp, issuer_key_hash)
+
+def mv_file(fromfn, tofn):
+    shutil.move(fromfn, tofn)
+
+def write_file(fn, sth):
+    tempname = fn + ".new"
+    open(tempname, 'w').write(json.dumps(sth))
+    mv_file(tempname, fn)
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
new file mode 100644
index 0000000..d84be8e
--- /dev/null
+++ b/monitor/josef_monitor.py
@@ -0,0 +1,492 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-     
+
+import time
+import datetime
+import base64
+import argparse
+import errno
+from certtools import *
+
+NAGIOS_OK = 0
+NAGIOS_WARN = 1
+NAGIOS_CRIT = 2
+NAGIOS_UNKNOWN = 3
+
+DEFAULT_CUR_FILE = 'all-sth.json'
+DEFAULT_CERT_FILE = "plausible_cert_data.json"
+
+base_urls = [
+    "https://plausible.ct.nordu.net/",
+    # "https://ct1.digicert-ct.com/log/", 
+    # "https://ct.izenpe.com/",
+    # "https://log.certly.io/", 
+    # "https://ct.googleapis.com/aviator/", 
+    # "https://ct.googleapis.com/pilot/",
+    # "https://ct.googleapis.com/rocketeer/",
+    # "https://ctlog.api.venafi.com/",
+    # "https://ct.ws.symantec.com/",
+]
+
+parser = argparse.ArgumentParser(description="")
+parser.add_argument('--audit', action='store_true', help="run lightweight auditor verifying consistency in STH")
+parser.add_argument('--monitor', action='store_true', help="run full monitoring for all logs")
+parser.add_argument('--audit2', action='store_true', help="run medium-weight auditor verifying consistency in STH and inclusion proofs of new entries")
+parser.add_argument('--audit3', action='store_true', help="continously run medium-weight auditor verifying consistency in STH and inclusion proofs of new entries")
+parser.add_argument('--audit4', action='store_true', help="run one check on one server")
+parser.add_argument('--build-sth', action='store_true', help="get all entries and construct STH")
+parser.add_argument('--verify-index', default=None, help="Verify a specific index in all logs" )
+parser.add_argument('--host', default=None, help="Base URL for CT log")
+parser.add_argument('--roots', action='store_true', help="Check accepted root certificates for all logs" )
+parser.add_argument('--cur-sth',
+                    metavar='file',
+                    default=DEFAULT_CUR_FILE,
+                    help="File containing current STH (default=%s)" % DEFAULT_CUR_FILE)
+
+timings = {}
+errors = []
+
+monitored_domains = [
+    # "google.com",
+    # "preishelden.de",
+    # "liu.se",
+    # "nordu.net",
+    # "symantec.com",
+    # "sunet.se",
+    # ".se",
+]
+
+class UTC(datetime.tzinfo):
+    def utcoffset(self, dt):
+      return datetime.timedelta(hours=0)
+    def dst(self, dt):
+        return datetime.timedelta(0)
+
+def reduce_layer(layer):
+    new_layer = []
+    while len(layer) > 1:
+        e1 = layer.pop(0)
+        e2 = layer.pop(0)
+        new_layer.append(internal_hash((e1,e2)))
+    return new_layer
+
+def reduce_tree(entries, layers):
+    if len(entries) == 0 and layers is []:
+        return [[hashlib.sha256().digest()]]
+  
+    layer_idx = 0
+    layers[layer_idx] += entries
+
+    while len(layers[layer_idx]) > 1:
+        if len(layers) == layer_idx + 1:
+            layers.append([])
+
+        layers[layer_idx + 1] += reduce_layer(layers[layer_idx]) 
+        layer_idx += 1
+    return layers
+
+def reduce_subtree_to_root(layers):
+    while len(layers) > 1:
+        if len(layers[1]) == 0:
+            layers[1] = layers[0]
+        else:
+            layers[1] += next_merkle_layer(layers[0])
+        del layers[0]
+
+    if len(layers[0]) > 1:
+        return next_merkle_layer(layers[0])
+    return layers[0]
+
+def fetch_all_sth():
+    sths = {}
+    for base_url in base_urls:
+        # Fetch STH
+        try:
+            sths[base_url] =  get_sth(base_url)
+        except:
+            sths[base_url] = None
+            error_str = time.strftime('%H:%M:%S') + " ERROR: Failed to retrieve STH from " + base_url
+            print error_str
+            errors.append(error_str)
+            continue
+
+        # Check signature on the STH
+        try:
+            # check_sth_signature(base_url, sths[base_url], logkeys[base_url])
+            check_sth_signature(base_url, sths[base_url], None)
+        except:
+            error_str = time.strftime('%H:%M:%S') + " ERROR: Could not verify signature from " + base_url
+            print error_str
+            errors.append(error_str)
+            continue
+
+    return sths
+
+def verify_progress(old, new):
+    print "Verifying progress"
+    try:
+        for url in new:
+            if new and old and new[url] and old[url]:
+                if new[url]["tree_size"] == old[url]["tree_size"]:
+                    if old[url]["sha256_root_hash"] != new[url]["sha256_root_hash"]:
+                        errors.append(time.strftime('%H:%M:%S') + " CRITICAL: root hash is different for same tree size in " + url)
+                elif new[url]["tree_size"] < old[url]["tree_size"]:
+                        errors.append(time.strftime('%H:%M:%S') + " CRITICAL: new tree smaller than previous tree (%d < %d)" % \
+                          (new[url]["tree_size"], old[url]["tree_size"]))
+            if new[url]:
+                age = time.time() - new[url]["timestamp"]/1000
+                sth_time = datetime.datetime.fromtimestamp(new[url]['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
+                roothash = new[url]['sha256_root_hash']
+                if age > 24 * 3600:
+                    errors.append(time.strftime('%H:%M:%S') + " CRITICAL: %s is older than 24h: %s UTC" % (url, sth_time))
+                elif age > 12 * 3600:
+                    errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 12h: %s UTC" % (url, sth_time))
+                elif age > 6 * 3600:
+                    errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 6h: %s UTC" % (url, sth_time))
+                # elif age > 2 * 3600:
+                #     errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 2h: %s UTC" % (url, sth_time))
+    except:
+        print time.strftime('%H:%M:%S') + " ERROR: Failed to verify progress for " + url
+
+
+def verify_consistency(old, new):
+    for url in old:
+        try:
+            if old[url] and new[url] and old[url]["tree_size"]!= new[url]["tree_size"]:
+                consistency_proof = get_consistency_proof(url, old[url]["tree_size"], new[url]["tree_size"])
+                decoded_consistency_proof = []
+                for item in consistency_proof:
+                    decoded_consistency_proof.append(base64.b64decode(item))
+                res = verify_consistency_proof(decoded_consistency_proof, old[url]["tree_size"], new[url]["tree_size"], old[url]["sha256_root_hash"])
+                
+                if old[url]["sha256_root_hash"] != str(base64.b64encode(res[0])):
+                    print time.strftime('%H:%M:%S') + " Verification of old hash failed! " + old[url]["sha256_root_hash"], str(base64.b64encode(res[0]))
+                    errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to verify consistency for " + url + ", tree size " + old[url]["tree_size"])
+                elif new[url]["sha256_root_hash"] != str(base64.b64encode(res[1])):
+                    print time.strftime('%H:%M:%S') + " Verification of new hash failed! " + new[url]["sha256_root_hash"], str(base64.b64encode(res[1]))
+                    errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to verify consistency for " + url + ", tree size " + new[url]["tree_size"])
+                else:
+                    print time.strftime("%H:%M:%S") + " New STH from " + url + ", timestamp: " + \
+                        str(new[url]["timestamp"]) + ", size: " + str(new[url]["tree_size"]) + "...OK."
+
+        except:
+          print "ERROR: Could not verify consistency for " + url
+
+def verify_inclusion_all(old, new):
+    for url in old:
+        try:
+            if old[url] and new[url]:
+                if old[url]["tree_size"]!= new[url]["tree_size"]:
+                    entries = []
+
+                    while len(entries) + old[url]["tree_size"]!= new[url]["tree_size"]:
+                        entries += get_entries(url, str(int(old[url]["tree_size"]) + len(entries)), new[url]["tree_size"] -1)["entries"]
+                        print "Got " + str(len(entries)) + " entries..."
+
+                    success = True
+                    for i in entries:
+                        h = get_leaf_hash(base64.b64decode(i["leaf_input"]))
+                        if not verify_inclusion_by_hash(url, h):
+                            success = False
+
+                    if success:
+                        print time.strftime("%H:%M:%S") + " Verifying inclusion for " + str(len(entries)) + " new entries in " + url + " ...OK"
+                    else:
+                        print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url
+                        errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url)
+        except:
+            print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url
+            errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url)
+
+def check_domain(raw_entry, log=None):
+    orig_entry = extract_original_entry(raw_entry)
+    cert_info = my_get_cert_info(orig_entry[0][0])
+    if log:
+        cert_info["log"] = log[8:-1] # strip generic URL stuff
+    return cert_info
+
+
+def fetch_and_increment_subtree(old_sth, new_sth_in, subtree, base_url):
+    try:
+        print "Initial hash:", hash(str(subtree))
+        sth = old_sth[base_url]
+        new_sth = new_sth_in[base_url]
+        idx = sth["tree_size"]
+        tmp_tree = list(subtree)
+        print "tmp hash:", hash(str(tmp_tree))
+
+        while idx < new_sth["tree_size"]:
+            pre_size = idx
+            entries = get_entries(base_url, idx, new_sth["tree_size"]-1)["entries"]
+            new_leafs = []
+            tmp_cert_data = []
+            for item in entries:
+                tmp_cert_data.append(check_domain(item, base_url))
+                new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
+            append_file(DEFAULT_CERT_FILE, tmp_cert_data)
+            idx += len(new_leafs)
+            print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " \
+                + str(idx -1) + " (" + str(len(new_leafs)) +" entries) from " + base_url
+            
+            print "Before reduction:", hash(str(tmp_tree))
+            res_tree = reduce_tree(new_leafs, tmp_tree)
+            print "After reduction:", hash(str(res_tree))
+
+    except:
+        print "Failed to build subtree :("
+
+    if subtree == res_tree:
+        print "Final subtree hash", hash(str(subtree))
+        print "Final restree hash", hash(str(res_tree))
+    return res_tree
+
+def fetch_and_build_subtree(old_sth, base_url):
+    try:
+        sth = old_sth[base_url]
+        subtree = [[]]
+        idx = 0
+
+        while idx < sth["tree_size"]:
+            pre_size = idx
+            entries = get_entries(base_url, idx, sth["tree_size"]-1)["entries"]
+            new_leafs = []
+            tmp_cert_data = []
+            for item in entries:
+                tmp_cert_data.append(check_domain(item, base_url))
+                new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
+            idx += len(new_leafs)
+            append_file(DEFAULT_CERT_FILE, tmp_cert_data)
+            print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx -1) + " from " + base_url
+            subtree = reduce_tree(new_leafs, subtree)
+
+    except:
+        print "Failed to build subtree :("
+
+    return subtree
+
+
+def verify_subtree(old_sth, subtree, base_url):
+    try:
+        sth = old_sth[base_url]
+        tmp = list(subtree)
+        root = base64.b64encode(reduce_subtree_to_root(tmp)[0])
+
+        if root == sth["sha256_root_hash"]:
+            print time.strftime('%H:%M:%S') + " Verifying root hashes for " + base_url + "...OK."
+        else:
+            print time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hashes! STH root: " \
+                + sth["sha256_root_hash"] + ", Tree root: " + root
+            # errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hash for " 
+            #     + base_url + ", tre size " + sth["tree_size"])
+    except:
+        print time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url
+        # errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url)
+
+def verify_inclusion_by_hash(base_url, leaf_hash):
+    try: 
+        tmp_sth = get_sth(base_url)
+        proof = get_proof_by_hash(base_url, leaf_hash, tmp_sth["tree_size"])
+
+        decoded_inclusion_proof = []
+        for item in proof["audit_path"]:
+            decoded_inclusion_proof.append(base64.b64decode(item))
+        
+        root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, proof["leaf_index"], tmp_sth["tree_size"], leaf_hash))
+
+        if tmp_sth["sha256_root_hash"] == root:
+            return True
+        else:
+            print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(proof["leaf_index"]) + " in " + base_url
+            errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(proof["leaf_index"]) + " in " + base_url)
+            return False
+    except:
+        print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for hashed entry in " + base_url
+        errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for hashed entry in " + base_url)
+        return False
+
+def verify_inclusion_by_index(base_url, index):
+    try: 
+        tmp_sth = get_sth(base_url)
+        proof = get_proof_by_index(base_url, index, tmp_sth["tree_size"])
+
+        decoded_inclusion_proof = []
+        for item in proof["audit_path"]:
+            decoded_inclusion_proof.append(base64.b64decode(item))
+
+        root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, index, tmp_sth["tree_size"], get_leaf_hash(base64.b64decode(proof["leaf_input"]))))
+
+        if tmp_sth["sha256_root_hash"] == root:
+            print time.strftime('%H:%M:%S') + " Verifying inclusion for entry " + str(index) + " in " + base_url + "...OK."
+        else:
+            print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url
+            errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url)
+    except:
+        print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url
+        errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url)
+
+def get_proof_by_index(baseurl, index, tree_size):
+    try:
+        params = urllib.urlencode({"leaf_index":index,
+                                   "tree_size":tree_size})
+        result = \
+          urlopen(baseurl + "ct/v1/get-entry-and-proof?" + params).read()
+        return json.loads(result)
+    except urllib2.HTTPError, e:
+        print "ERROR:", e.read()
+        sys.exit(0)
+
+def get_all_roots(base_url):
+    result = urlopen(base_url + "ct/v1/get-roots").read()
+    certs = json.loads(result)["certificates"]
+    print time.strftime('%H:%M:%S') + " Received " + str(len(certs)) + " certs from " + base_url
+
+    for accepted_cert in certs:
+        subject = get_cert_info(base64.decodestring(accepted_cert))["subject"]
+        issuer = get_cert_info(base64.decodestring(accepted_cert))["issuer"]
+        if subject == issuer:
+            root_cert = base64.decodestring(accepted_cert)
+            print get_cert_info(root_cert)["subject"]
+
+def print_errors(errors):
+    print "Encountered " + str(len(errors)) + " errors:"
+    for item in errors:
+        print item
+
+def print_timings(timings):
+    for item in timings:
+        m,s = divmod(timings[item]["longest"], 60)
+        h,m = divmod(m, 60)
+        print item + " last seen " + datetime.datetime.fromtimestamp(int(timings[item]["last"])/1000).strftime('%Y-%m-%d %H:%M:%S') \
+            + " longest between two STH: " + str(int(h)) + "h " + str(int(m)) + "m "# + str(int(s)) + "s."
+
+def read_sth(fn):
+    try:
+        f = open(fn)
+    except IOError, e:
+        if e.errno == errno.ENOENT:
+            return None
+        raise e
+    return json.loads(f.read())
+
+def write_file(fn, content):
+    tempname = fn + ".new"
+    open(tempname, 'w').write(json.dumps(content))
+    mv_file(tempname, fn)
+    # print "wrote " + fn
+
+def append_file(fn, content):
+    with open(fn, 'a') as f:
+        for item in content:
+            try: 
+                f.write(json.dumps(item) + "\n")
+            except:
+                # print "failed to write " + str(item)
+                pass
+
+def main(args):
+    
+    # print time.strftime("%H:%M:%S") + " Starting..."
+    if args.verify_index is None and not args.build_sth and not args.audit and not args.audit2 \
+        and not args.audit3 and not args.audit4 and not args.roots and not args.monitor:
+        
+        print time.strftime('%H:%M:%S') + " Nothing to do."
+        return
+    elif args.audit4:
+        pass
+    else:
+        sth = fetch_all_sth()
+
+    if args.verify_index is not None:
+        for url in base_urls:
+            verify_inclusion_by_index(url, int(args.verify_index))
+
+    if args.roots:
+        print time.strftime('%H:%M:%S') + " Getting accepted Root Certs from all logs..."
+        for url in base_urls:
+            get_all_roots(url)
+
+
+    if args.build_sth:
+        print time.strftime('%H:%M:%S') + " Building trees from entries. This may take a while, go get coffee or something..."
+        for base_url in base_urls:
+            subtree = fetch_and_build_subtree(sth, base_url)
+            verify_subtree(sth, subtree, base_url)
+        # fetch_and_build_tree(sth, base_urls[2])
+
+    if args.audit:
+        print time.strftime('%H:%M:%S') + " Running auditor1 for " +str(len(base_urls)) + " logs..."
+        old_sth = read_sth(args.cur_sth)
+        if old_sth:
+            verify_consistency(old_sth, sth)
+        else: 
+            print "No old sth found..."
+        write_file(args.cur_sth, sth)
+
+
+    if args.audit3:
+        print time.strftime('%H:%M:%S') + " Running auditor3 for " +str(len(base_urls)) + " logs..."
+        while True:
+            time.sleep(30)
+            new_sth = fetch_all_sth()
+            verify_consistency(sth, new_sth)
+            verify_inclusion_all(sth, new_sth)
+            sth = new_sth
+
+    if args.audit2:
+        print time.strftime('%H:%M:%S') + " Running auditor2 for " +str(len(base_urls)) + " logs..."
+        old_sth = read_sth(args.cur_sth)
+            # print "Verifying progress..."
+        verify_progress(old_sth, sth)
+        if old_sth:
+            print "Verifying consistency..."
+            verify_consistency(old_sth, sth)
+            print "Verifying inclusion..."
+            verify_inclusion_all(old_sth, sth)
+        write_file(args.cur_sth, sth)
+    
+    if args.monitor:
+        all_subtrees = {}
+        print time.strftime('%H:%M:%S') + " Building trees from entries. This may take a while, go get coffee or something..."
+        for url in base_urls:
+            all_subtrees[url] = fetch_and_build_subtree(sth, url)
+            verify_subtree(sth, all_subtrees[url], url)
+
+        while True:
+            time.sleep(30)
+            new_sth = fetch_all_sth()
+            for url in base_urls:
+                if url in sth and url in new_sth and sth[url]["tree_size"] != new_sth[url]["tree_size"]:
+                    # print "Len before:", len(all_subtrees[url])
+                    all_subtrees[url] = fetch_and_increment_subtree(sth, new_sth, all_subtrees[url], url)
+                    # print "Len between:", len(all_subtrees[url])
+                    verify_subtree(new_sth, all_subtrees[url], url)
+                    # print "Len after:", len(all_subtrees[url])
+                    print "Old sth:", sth[url]["sha256_root_hash"]
+                    print "New sth:", new_sth[url]["sha256_root_hash"]
+            sth = new_sth
+
+
+if __name__ == '__main__':
+    # try:
+        main(parser.parse_args())
+        if len(errors) == 0:
+            print time.strftime('%H:%M:%S') + " Everything OK."
+            sys.exit(NAGIOS_OK)
+        else:
+            # print "errors found!"
+            print_errors(errors)
+            sys.exit(NAGIOS_WARN)
+    # except:
+    #     pass
+    # finally:
+    #     # print_timings(timings)
+    #     print_errors(errors)
+
+
+
+
+
+
+
+
+
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
new file mode 100755
index 0000000..bf415f7
--- /dev/null
+++ b/monitor/josef_reader.py
@@ -0,0 +1,60 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-     
+
+import sys
+from certtools import *
+import argparse
+
+
+
+parser = argparse.ArgumentParser(description="")
+parser.add_argument('--domain', default=None, help="RTFM")
+parser.add_argument('--exclude-expired', action='store_true', help="RTFM")
+
+args = parser.parse_args()
+
+monitored_domains = [
+    "google.com",
+    "preishelden.de",
+    "liu.se",
+    "nordu.net",
+    "symantec.com",
+]
+
+
+
+# data = []
+f = open("plausible_cert_data.json")
+for line in f:
+    tmp = json.loads(line)
+    try: 
+        success = True
+
+        if args.domain:
+            if args.domain in tmp["subject"].split("CN=")[1] or \
+            args.domain in tmp["SAN"]:
+                pass
+            else:
+                success = False
+
+        if args.exclude_expired:
+            print "EXCLUDE EXPIRED NOT IMPLEMENTED YET"
+
+
+        if success:
+            print tmp["subject"].split("CN=")[1] + " certified by " + tmp["issuer"].split("CN=")[1]
+    except:
+        pass
+
+f.close()
+
+# for item in data[10000:]:
+#     try:
+#         s = item["subject"].split("CN=")[1]
+#         print "\n" + s
+#         print item["SAN"]
+#     except:
+#         pass
+
+# print "\nTotal entries: " + str(len(data))
+
diff --git a/monitor/monitor.cfg b/monitor/monitor.cfg
new file mode 100644
index 0000000..1d17802
--- /dev/null
+++ b/monitor/monitor.cfg
@@ -0,0 +1,7 @@
+# CONFIG FILE FOR MONITOR
+
+# LOGS
+
+# CHECKS
+
+# OUTPUT 
diff --git a/tools/certkeys.py b/tools/certkeys.py
index dd0570f..f7c83b1 100644
--- a/tools/certkeys.py
+++ b/tools/certkeys.py
@@ -4,10 +4,6 @@ publickeys = {
     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTD"
     "M0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==",
 
-    "https://flimsy.ct.nordu.net/":
-    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9"
-    "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==",
-
     "https://plausible.ct.nordu.net/":
     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUB"
 	"c8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ==",
diff --git a/tools/josef_experimental.py b/tools/josef_experimental.py
index 383e385..7f27945 100755
--- a/tools/josef_experimental.py
+++ b/tools/josef_experimental.py
@@ -136,26 +136,30 @@ monitored_domains = [
     "symantec.com",
 ]
 
-# data = read_sth("plausible_cert_data.json")
-# ss = []
-# for item in data:
-#     try:
-#         s = item["subject"].split("CN=")[1]
-#         print s
-#     except:
-#         # if not item["subject"] in ss:
-#         #     print item["subject"]
-#         #     ss.append(item["subject"])
-#         pass
-
-# print "\nTotal entries: " + str(len(data))
-
-base_url = base_urls[0]
-
-entries = get_entries(base_url, 11, 11)["entries"]
-for item in entries:
-    orig_entry = extract_original_entry(item)
-    cert_info = my_get_cert_info(orig_entry[0][0])
+data = []
+# data = read_sth("cert_data.json")
+f = open("cert_data.json")
+for line in f:
+    data.append(json.loads(line))
+ss = []
+for item in data:
+    try:
+        s = item["subject"].split("CN=")[1]
+        print s
+    except:
+        # if not item["subject"] in ss:
+        #     print item["subject"]
+        #     ss.append(item["subject"])
+        pass
+
+print "\nTotal entries: " + str(len(data))
+
+# base_url = base_urls[0]
+
+# entries = get_entries(base_url, 11, 11)["entries"]
+# for item in entries:
+#     orig_entry = extract_original_entry(item)
+#     cert_info = my_get_cert_info(orig_entry[0][0])
     # prev = ""
     # res = {}
     # for line in cert_info:
@@ -172,4 +176,4 @@ for item in entries:
     #         res["not_after"] = line.split(": ")[1]
 
     #     prev = line
-    print cert_info
+    # print cert_info
diff --git a/tools/josef_experimental_auditor.py b/tools/josef_experimental_auditor.py
index 78a3fe2..7efd2dc 100755
--- a/tools/josef_experimental_auditor.py
+++ b/tools/josef_experimental_auditor.py
@@ -14,16 +14,17 @@ NAGIOS_CRIT = 2
 NAGIOS_UNKNOWN = 3
 
 DEFAULT_CUR_FILE = 'all-sth.json'
+DEFAULT_CERT_FILE = "cert_data.json"
 
 base_urls = [
     # "https://plausible.ct.nordu.net/",
     # "https://ct1.digicert-ct.com/log/", 
-    # "https://ct.izenpe.com/",
+    "https://ct.izenpe.com/",
     # "https://log.certly.io/", 
     # "https://ct.googleapis.com/aviator/", 
     # "https://ct.googleapis.com/pilot/",
     # "https://ct.googleapis.com/rocketeer/",
-    # "https://ctlog.api.venafi.com/",
+    "https://ctlog.api.venafi.com/",
     "https://ct.ws.symantec.com/",
 ]
 
@@ -55,14 +56,6 @@ monitored_domains = [
     # ".se",
 ]
 
-# cert_data = []
-
-# class cert(subject, issuer, log):
-#     def __init__(self):
-#         self.subject = subject
-#         self.issuer = issuer
-        # self.logs = [log]
-
 class UTC(datetime.tzinfo):
     def utcoffset(self, dt):
       return datetime.timedelta(hours=0)
@@ -215,9 +208,12 @@ def check_domain(raw_entry, log=None):
 
 def fetch_and_increment_subtree(old_sth, new_sth_in, subtree, base_url):
     try:
+        print "Initial hash:", hash(str(subtree))
         sth = old_sth[base_url]
         new_sth = new_sth_in[base_url]
         idx = sth["tree_size"]
+        tmp_tree = list(subtree)
+        print "tmp hash:", hash(str(tmp_tree))
 
         while idx < new_sth["tree_size"]:
             pre_size = idx
@@ -227,16 +223,22 @@ def fetch_and_increment_subtree(old_sth, new_sth_in, subtree, base_url):
             for item in entries:
                 tmp_cert_data.append(check_domain(item, base_url))
                 new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
-            append_file("cert_data.json", tmp_cert_data)
+            append_file(DEFAULT_CERT_FILE, tmp_cert_data)
             idx += len(new_leafs)
             print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " \
                 + str(idx -1) + " (" + str(len(new_leafs)) +" entries) from " + base_url
-            subtree = reduce_tree(new_leafs, subtree)
+            
+            print "Before reduction:", hash(str(tmp_tree))
+            res_tree = reduce_tree(new_leafs, tmp_tree)
+            print "After reduction:", hash(str(res_tree))
 
     except:
         print "Failed to build subtree :("
 
-    return subtree 
+    if subtree == res_tree:
+        print "Final subtree hash", hash(str(subtree))
+        print "Final restree hash", hash(str(res_tree))
+    return res_tree
 
 def fetch_and_build_subtree(old_sth, base_url):
     try:
@@ -253,8 +255,8 @@ def fetch_and_build_subtree(old_sth, base_url):
                 tmp_cert_data.append(check_domain(item, base_url))
                 new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
             idx += len(new_leafs)
-            append_file("cert_data.json", tmp_cert_data)
-            print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx) + " from " + base_url
+            append_file(DEFAULT_CERT_FILE, tmp_cert_data)
+            print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx -1) + " from " + base_url
             subtree = reduce_tree(new_leafs, subtree)
 
     except:
@@ -376,7 +378,7 @@ def append_file(fn, content):
     with open(fn, 'a') as f:
         for item in content:
             try: 
-                f.write(json.dumps(item))
+                f.write(json.dumps(item) + "\n")
             except:
                 # print "failed to write " + str(item)
                 pass
@@ -454,8 +456,13 @@ def main(args):
             new_sth = fetch_all_sth()
             for url in base_urls:
                 if url in sth and url in new_sth and sth[url]["tree_size"] != new_sth[url]["tree_size"]:
+                    # print "Len before:", len(all_subtrees[url])
                     all_subtrees[url] = fetch_and_increment_subtree(sth, new_sth, all_subtrees[url], url)
+                    # print "Len between:", len(all_subtrees[url])
                     verify_subtree(new_sth, all_subtrees[url], url)
+                    # print "Len after:", len(all_subtrees[url])
+                    print "Old sth:", sth[url]["sha256_root_hash"]
+                    print "New sth:", new_sth[url]["sha256_root_hash"]
             sth = new_sth