path: root/monitor/josef_monitor.py

#!/usr/bin/python
# -*- coding: utf-8 -*-  

# Selected dependencies
# python-dev (apt)
# pycrypto (pip)
# leveldb (pip)

import time
import datetime
import base64
import argparse
import errno
from copy import deepcopy
from josef_lib import *
from josef_reader import monitored_domain
from josef_leveldb import db_add_certs, db_open
import os.path

# Import from config file
if os.path.isfile("monitor_conf.py"):
    from monitor_conf import ctlogs, OUTPUT_DIR, INTERVAL, DEFAULT_CERT_FILE, DB_PATH, MONITORED_DOMAINS, DOMAINS_FILE
else:
    print "Config file not found!"
    sys.exit()

if not os.path.exists(OUTPUT_DIR):
    os.makedirs(OUTPUT_DIR)

if not os.path.exists(DB_PATH):
    os.makedirs(DB_PATH)


parser = argparse.ArgumentParser(description="")

class ctlog:
    def __init__(self, name, url, key, log_id=None):
        self.name = name
        self.url = url
        self.key = key
        self.log_id = log_id
        self.logfile = OUTPUT_DIR + name + ".log"
        self.savefile = OUTPUT_DIR + name + "-state-info.json"
        self.subtree = [[]]
        self.sth = None
        self.entries = 0
        self.root_hash = None

        self.log("Starting monitor")


    def incremental_build(self):
        # Keeps state current during build, partial builds are possible.
        self.sth = get_sth(self.url)
        start_size = self.entries
        while self.entries < self.sth["tree_size"]:
            tmp_size = self.entries
            self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
            if tmp_size != self.entries:
                self.log("Got entries " + str(tmp_size) + " to " \
                + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1))

        if self.entries != start_size:
            if verify_subtree(self.sth, self.subtree, self.url):
                self.log("Successfully build tree with " + str(self.entries - start_size) + \
                    " new entries. Size: " + str(self.entries))
            else:
                self.log("ERROR Failed to build tree from entries.")


    def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]):
        new_leafs = []
        if first <= last:
            entries = get_entries(url, first, last)["entries"]
            tmp_cert_data = []
            for item in entries:
                tmp_data = check_domain(item, url)
                entry_hash = get_leaf_hash(base64.b64decode(item["leaf_input"]))
                if tmp_data:
                    tmp_data["leaf_hash"] = base64.b64encode(entry_hash)
                    tmp_cert_data.append(tmp_data)
                new_leafs.append(entry_hash)
            if DB_PATH:
                db_add_certs(DB_PATH, tmp_cert_data)
            if DEFAULT_CERT_FILE:
                append_file(DEFAULT_CERT_FILE, tmp_cert_data)
            subtree = reduce_tree(new_leafs, subtree)
        return subtree, len(new_leafs) + first


    def to_dict(self):
        d = {}
        d["entries"] = self.entries
        d["subtree"] = encode_tree(self.subtree)
        d["sth"] = self.sth
        return d

    def save(self):
        self.log("Saving state to file")
        open(self.savefile, 'w').write(json.dumps(self.to_dict()))

    def load(self):
        self.log("Loading state from file")
        try:
            f = open(self.savefile)
            s = f.read()
            d = json.loads(s)
            self.subtree = decode_tree(d["subtree"])
            self.sth = d["sth"]
            self.entries = d["entries"]
        except IOError, e:
            if e.errno == errno.ENOENT:
                return None
            raise e

    def log(self, string):
        s = time.strftime('%Y-%m-%d, %H:%M:%S') + " " + string
        with open(self.logfile, 'a') as f:
            f.write(s + "\n")
            f.close()

    def update_sth(self):
        new_sth = get_sth(self.url)

        try:
            check_sth_signature(self.url, new_sth, None)
        except:
            self.log("ERROR: Could not verify STH signature")
            print "ERROR: Could not verify STH signature from " + self.url

        sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
        if new_sth["timestamp"] != self.sth["timestamp"]:
            self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time)
            self.sth = new_sth


    def update_roots(self):
        roots = get_all_roots(self.url)
        new_root_hash = str(hash(str(roots)))
        
        if new_root_hash != self.root_hash:
            self.root_hash = new_root_hash
            cert_dir = OUTPUT_DIR + self.name + "-roots"
            if not os.path.exists(cert_dir):
                os.makedirs(cert_dir)

            hash_list = []
            for cert in roots:
                h = str(hash(str(cert)))
                hash_list.append(h)

            loaded_list = os.listdir(cert_dir)

            added, removed = compare_lists(hash_list, loaded_list)

            if len(added) != 0:
                print str(len(added)) + " new roots found for " + self.name
            if len(removed) != 0:
                print str(len(removed)) + " roots removed for " + self.name
            
            for item in removed:
                data = open(cert_dir + "/" + item).read()

                root_cert = base64.decodestring(data)
                subject = get_cert_info(root_cert)["subject"]
                issuer = get_cert_info(root_cert)["issuer"]
                if subject == issuer:
                    print "Removed Root: " + item + ", " + subject
                    self.log("Removed Root: " + item + ", " + subject)
                else: 
                    print "WTF? Not a root..."

            for item in added:
                root_cert = base64.decodestring(roots[hash_list.index(item)])
                subject = get_cert_info(root_cert)["subject"]
                issuer = get_cert_info(root_cert)["issuer"]
                if subject == issuer:
                    print "New Root: " + item + ", " + subject
                    self.log("New Root: " + item + ", " + subject)
                else: 
                    print "WTF? Not a root..."

                fn = cert_dir + "/" + item
                tempname = fn + ".new"
                data = roots[hash_list.index(item)]
                open(tempname, 'w').write(data)
                mv_file(tempname, fn)


    def verify_progress(self, old):
        new = self.sth
        try:
            if new["tree_size"] == old["tree_size"]:
                if old["sha256_root_hash"] != new["sha256_root_hash"]:
                    s = time.strftime('%H:%M:%S') + " CRITICAL: root hash is different for same tree size"
                    self.log(s)
                    print s
            elif new["tree_size"] < old["tree_size"]:
                s = time.strftime('%H:%M:%S') + " CRITICAL: new tree smaller than previous tree (%d < %d)" % \
                  (new["tree_size"], old["tree_size"])
                self.log(s)
                print s
            else:
                age = time.time() - new["timestamp"]/1000
                sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
                roothash = new['sha256_root_hash']
                if age > 24 * 3600:
                    s = "CRITICAL: STH is older than 24h: %s UTC" % (sth_time)
                    self.log(s)
                    print s
                elif age > 12 * 3600:
                    s = "WARNING: STH is older than 12h: %s UTC" % (sth_time)
                    self.log(s)
                    # print s
                elif age > 6 * 3600:
                    s = "WARNING: STH is older than 6h: %s UTC" % (sth_time)
                    self.log(s)
                    # print s
        except:
            s = " ERROR: Failed to verify progress"
            self.log(s)
            print s


    def verify_consistency(self, old):
        new = self.sth
        # for url in old:
        try:
            if old["tree_size"]!= new["tree_size"]:
                consistency_proof = get_consistency_proof(self.url, old["tree_size"], new["tree_size"])
                decoded_consistency_proof = []
                for item in consistency_proof:
                    decoded_consistency_proof.append(base64.b64decode(item))
                res = verify_consistency_proof(decoded_consistency_proof, old["tree_size"], new["tree_size"], old["sha256_root_hash"])
                
                if old["sha256_root_hash"] != str(base64.b64encode(res[0])):
                    s = " Verification of old hash failed! " + \
                        old["sha256_root_hash"], str(base64.b64encode(res[0]))
                    self.log(s)
                    print s
                elif new["sha256_root_hash"] != str(base64.b64encode(res[1])):
                    s = " Verification of new hash failed! " + \
                        new["sha256_root_hash"], str(base64.b64encode(res[1]))
                    self.log(s)
                    print s
                # else:
                #     s = "New STH, timestamp: " + str(new["timestamp"]) + \
                #         ", size: " + str(new["tree_size"]) + "...OK."
                #     self.log(s)

        except:
            self.log("ERROR: Could not verify consistency!")
            print "ERROR: Could not verify consistency for " + self.url


def verify_inclusion_all(old, new):
    for url in old:
        try:
            if old[url] and new[url]:
                if old[url]["tree_size"]!= new[url]["tree_size"]:
                    entries = []

                    while len(entries) + old[url]["tree_size"]!= new[url]["tree_size"]:
                        entries += get_entries(url, str(int(old[url]["tree_size"]) + len(entries)), new[url]["tree_size"] -1)["entries"]
                        print "Got " + str(len(entries)) + " entries..."

                    success = True
                    for i in entries:
                        h = get_leaf_hash(base64.b64decode(i["leaf_input"]))
                        if not verify_inclusion_by_hash(url, h):
                            success = False

                    if success:
                        print time.strftime("%H:%M:%S") + " Verifying inclusion for " + str(len(entries)) + " new entries in " + url + " ...OK"
                    else:
                        print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url
                        errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url)
        except:
            print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url
            errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url)

def check_domain(raw_entry, log=None):
    orig_entry = extract_original_entry(raw_entry)
    try:
        cert_info = my_get_cert_info(orig_entry[0][0])
        if log:
            cert_info["log"] = log[8:-1] # strip generic URL stuff
        return cert_info
    except IndexError:
        return None

def verify_subtree(sth, subtree, base_url):
    try:
        tmp = deepcopy(subtree)
        root = base64.b64encode(reduce_subtree_to_root(tmp)[0])

        if root == sth["sha256_root_hash"]:
            # print time.strftime('%H:%M:%S') + " Verifying root hashes for " + base_url + "...OK."
            return True
        else:
            print time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hashes! STH root: " \
                + sth["sha256_root_hash"] + ", Tree root: " + root
            return False
    except:
        print time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url
        return False

def verify_inclusion_by_hash(base_url, leaf_hash):
    try: 
        tmp_sth = get_sth(base_url)
        proof = get_proof_by_hash(base_url, leaf_hash, tmp_sth["tree_size"])

        decoded_inclusion_proof = []
        for item in proof["audit_path"]:
            decoded_inclusion_proof.append(base64.b64decode(item))
        
        root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, proof["leaf_index"], tmp_sth["tree_size"], leaf_hash))

        if tmp_sth["sha256_root_hash"] == root:
            return True
        else:
            print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(proof["leaf_index"]) + " in " + base_url
            return False
    except:
        print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for hashed entry in " + base_url
        return False

def verify_inclusion_by_index(base_url, index):
    try: 
        tmp_sth = get_sth(base_url)
        proof = get_proof_by_index(base_url, index, tmp_sth["tree_size"])

        decoded_inclusion_proof = []
        for item in proof["audit_path"]:
            decoded_inclusion_proof.append(base64.b64decode(item))

        root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, index, tmp_sth["tree_size"], get_leaf_hash(base64.b64decode(proof["leaf_input"]))))

        if tmp_sth["sha256_root_hash"] == root:
            print time.strftime('%H:%M:%S') + " Verifying inclusion for entry " + str(index) + " in " + base_url + "...OK."
        else:
            print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url
            errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url)
    except:
        print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url
        errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url)

def get_proof_by_index(baseurl, index, tree_size):
    try:
        params = urllib.urlencode({"leaf_index":index,
                                   "tree_size":tree_size})
        result = \
          urlopen(baseurl + "ct/v1/get-entry-and-proof?" + params).read()
        return json.loads(result)
    except urllib2.HTTPError, e:
        print "ERROR:", e.read()
        sys.exit(0)

def get_all_roots(base_url):
    result = urlopen(base_url + "ct/v1/get-roots").read()
    certs = json.loads(result)["certificates"]
    print time.strftime('%H:%M:%S') + " Received " + str(len(certs)) + " certs from " + base_url
    return certs


def setup_domain_monitoring():
    monitored_domains = []
    try:
        with open(DOMAINS_FILE) as fp:
            for line in fp:
                tmp = json.loads(line)
                for domain in tmp:
                    md = monitored_domain(domain["url"])
                    md.load_entries(domain["entries"])
                    monitored_domains.append(md)
    except IOError:
        pass

    for md in MONITORED_DOMAINS: 
        tmp = monitored_domain(md)
        if not tmp in monitored_domains:
            print "New domain (not in file) " + md
            tmp.set()   
            monitored_domains.append(tmp)
    return monitored_domains

def main(args):
    global DB
    monitored_domains = setup_domain_monitoring()

    # Create logs
    logs = []
    try:
        for item in ctlogs:
            logs.append(ctlog(item["name"], item["url"], item["key"], item["id"]))

        print time.strftime('%H:%M:%S') + " Setting up monitor for " + str(len(logs)) + " logs..."
        # Set up state 
        for log in logs:
            if os.path.isfile(log.savefile):
                log.load()
        # Build what was not loaded
        # try:
        for log in logs:
            log.incremental_build()

        # Main loop: Monitor
        print time.strftime('%H:%M:%S') + " Running... (see logfiles for output)"
        while True:
            time.sleep(INTERVAL)
            for log in logs:
                old_sth = log.sth
                log.update_sth() # Should this be done is later checks fail? (reorder?)
                if old_sth["timestamp"] != log.sth["timestamp"]:
                    log.verify_progress(old_sth)
                    log.verify_consistency(old_sth)
                    log.incremental_build()

            for md in monitored_domains:
                md.update()

    # Normal exit oof the program
    except KeyboardInterrupt:
        print time.strftime('%H:%M:%S') + ' Received interrupt from user. Saving and exiting....'
        for log in logs:
            log.save()

        # Save info about monitored domains
        domain_dict = []
        for md in monitored_domains:
            domain_dict.append(md.to_dict())
        open(DOMAINS_FILE, 'w').write(json.dumps(domain_dict))

    # Something went horribly wrong!
    except Exception, err:
        print Exception, err
        for log in logs:
            log.save()

        # Save info about monitored domains
        domain_dict = []
        for md in monitored_domains:
            domain_dict.append(md.to_dict())
        open(DOMAINS_FILE, 'w').write(json.dumps(domain_dict))



if __name__ == '__main__':
    main(parser.parse_args())