path: root/monitor/josef_monitor.py

#!/usr/bin/python
# -*- coding: utf-8 -*-  

# Selected dependencies
# python-dev (apt)
# python-crypto (apt)
# leveldb (pip)

import time
import datetime
import base64
import argparse
import errno
from copy import deepcopy
from josef_lib import *
from josef_reader import monitored_domain
try:
    from josef_leveldb import db_add_certs, db_open
except:
    print "No database support found"
import os.path


parser = argparse.ArgumentParser(description="")
parser.add_argument('--config', default="monitor_conf.py")
args = parser.parse_args()

# Import from config file
if os.path.isfile(args.config):
    modules = map(__import__, [args.config[:-2]])
    CONFIG = modules[0]
    ERROR_STR = CONFIG.ERROR_STR
else:
    print "Config file not found!"
    ERROR_STR = "(local)ERROR: "
    sys.exit()



class ctlog:
    def __init__(self, name, url, key, log_id=None, build=True):
        self.name = name
        self.url = url
        self.key = key
        self.log_id = log_id
        self.logfile = CONFIG.OUTPUT_DIR + name + ".log"
        self.savefile = CONFIG.OUTPUT_DIR + name + "-state-info.json"
        self.subtree = [[]]
        self.sth = None
        self.entries = 0
        self.root_hash = None
        self.build = build

        if CONFIG.DB_PATH:
            self.dbdir = CONFIG.DB_PATH
            # self.dbdir = CONFIG.DB_PATH + name + "/"
            if not os.path.exists(self.dbdir):
                os.makedirs(self.dbdir)

        else:
            self.dbdir = None

        self.log("Starting monitor")


    def incremental_build(self):
        # Keeps state current during build, partial builds are possible.
        try:
            self.sth = get_sth(self.url)
        except Exception, e:
            self.log("Failed to fetch STH. " + str(e))
            return

        if self.build:
            start_size = self.entries
            # try:
            while self.entries < self.sth["tree_size"]:
                tmp_size = self.entries
                try:    
                    self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
                except Exception, e:
                    print ERROR_STR  + "Failed fetch and increment for " + self.name
                    self.log(ERROR_STR + "Failed fetch and increment tree. Current Size: " + str(self.entries) + " Sth: " + str(self.sth) + " Error: " + str(e))
                    return

                if tmp_size != self.entries:
                    self.log("Got entries " + str(tmp_size) + " to " \
                    + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1))

            if self.entries != start_size:
                if verify_subtree(self.sth, self.subtree, self.url):
                    pass
                else:
                    self.log(ERROR_STR + "Failed to verify newly built subtree!")
            # except Exception, e:
            #     print "Failed incremental build for " + self.name
            #     self.log(ERROR_STR + "Failed incremental build. Error: " + str(e))

    def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]):
        new_leafs = []
        if first <= last:
            entries = get_entries(url, first, last)["entries"]
            tmp_cert_data = []
            for item in entries:
                tmp_data = check_domain(item, url)
                entry_hash = get_leaf_hash(base64.b64decode(item["leaf_input"]))
                if tmp_data:
                    tmp_data["leaf_hash"] = base64.b64encode(entry_hash)
                    tmp_cert_data.append(tmp_data)
                new_leafs.append(entry_hash)
            if self.dbdir:
                db_add_certs(self.dbdir, tmp_cert_data)
            if CONFIG.DEFAULT_CERT_FILE:
                append_file(CONFIG.DEFAULT_CERT_FILE, tmp_cert_data)
            subtree = reduce_tree(new_leafs, subtree)
        return subtree, len(new_leafs) + first

    def to_dict(self):
        d = {}
        d["entries"] = self.entries
        d["subtree"] = encode_tree(self.subtree)
        d["sth"] = self.sth
        return d

    def save(self):
        self.log("Saving state to file")
        open(self.savefile, 'w').write(json.dumps(self.to_dict()))

    def load(self):
        self.log("Loading state from file")
        try:
            f = open(self.savefile)
            s = f.read()
            d = json.loads(s)
            self.subtree = decode_tree(d["subtree"])
            self.sth = d["sth"]
            self.entries = d["entries"]
        except IOError, e:
            if e.errno == errno.ENOENT:
                return None
            raise e

    def log(self, string):
        s = time.strftime('%Y-%m-%d %H:%M:%S') + " " + string
        s = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + " " + string
        with open(self.logfile, 'a') as f:
            f.write(s + "\n")
            f.close()

    def update_sth(self):
        try:
            new_sth = get_sth(self.url)
        except Exception, e:
            self.log(ERROR_STR + "Failed to fetch STH. " +str(e))
            return

        try:
            check_sth_signature(self.url, new_sth, base64.b64decode(self.key))
        except:
            self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth))
            print ERROR_STR + "Could not verify STH signature from " + self.url

        sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
        if new_sth["timestamp"] != self.sth["timestamp"]:
            self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time)
            self.sth = new_sth

    def get_all_roots(self):
        result = urlopen(self.url + "ct/v1/get-roots").read()
        certs = json.loads(result)["certificates"]
        return certs

    def update_roots(self):
        try:
            roots = self.get_all_roots()
        except Exception, e:
            self.log(ERROR_STR + "Failed to fetch roots. " + str(e))
            return

        new_root_hash = str(hash(str(roots)))
        
        if new_root_hash != self.root_hash:
            self.root_hash = new_root_hash
            cert_dir = CONFIG.OUTPUT_DIR + self.name + "-roots"
            if not os.path.exists(cert_dir):
                os.makedirs(cert_dir)

            hash_list = []
            for cert in roots:
                h = str(hash(str(cert)))
                hash_list.append(h)

            loaded_list = os.listdir(cert_dir)

            added, removed = compare_lists(hash_list, loaded_list)

            if len(added) != 0:
                for item in added:
                    root_cert = base64.decodestring(roots[hash_list.index(item)])
                    subject = get_cert_info(root_cert)["subject"]
                    issuer = get_cert_info(root_cert)["issuer"]
                    if subject == issuer:
                        self.log("Added Root: " + item + ", " + subject)
                    else: 
                        self.log(ERROR_STR + "Non self-signed root cert added! Subject:" + subject + " Issuer:" + issuer)

                    fn = cert_dir + "/" + item
                    tempname = fn + ".new"
                    data = roots[hash_list.index(item)]
                    open(tempname, 'w').write(data)
                    mv_file(tempname, fn)
            

            if len(removed) != 0:
                for item in removed:
                    data = open(cert_dir + "/" + item).read()
                    root_cert = base64.decodestring(data)
                    subject = get_cert_info(root_cert)["subject"]
                    issuer = get_cert_info(root_cert)["issuer"]
                    if subject == issuer:
                        self.log("Removed Root: " + item + ", " + subject)
                    else: 
                        self.log(ERROR_STR + "Non self-signed root cert removed! Subject:" + subject + " Issuer:" + issuer)

    def verify_progress(self, old):
        new = self.sth
        try:
            if new["tree_size"] == old["tree_size"]:
                if old["sha256_root_hash"] != new["sha256_root_hash"]:
                    print ERROR_STR + "Root hash is different for same tree size in " + self.name
                    self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new))
            elif new["tree_size"] < old["tree_size"]:
                print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \
                  (new["tree_size"], old["tree_size"], self.name)
                self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new))
            else:
                age = time.time() - new["timestamp"]/1000
                sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
                roothash = new['sha256_root_hash']
                if age > 24 * 3600:
                    s = ERROR_STR + "STH is older than 24h: %s UTC" % (sth_time)
                    self.log(s + str(new))
                    print s
                elif age > 12 * 3600:
                    s = "WARNING: STH is older than 12h: %s UTC" % (sth_time)
                    self.log(s)
                elif age > 6 * 3600:
                    s = "WARNING: STH is older than 6h: %s UTC" % (sth_time)
                    self.log(s)
        except Exception, e:
            self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e))
            print "Failed to verify progress in " + self.name

    def verify_consistency(self, old):
        new = self.sth
        try:
            if old["tree_size"]!= new["tree_size"]:
                consistency_proof = get_consistency_proof(self.url, old["tree_size"], new["tree_size"])
                decoded_consistency_proof = []
                for item in consistency_proof:
                    decoded_consistency_proof.append(base64.b64decode(item))
                res = verify_consistency_proof(decoded_consistency_proof, old["tree_size"], new["tree_size"], old["sha256_root_hash"])
                
                if old["sha256_root_hash"] != str(base64.b64encode(res[0])):
                    self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \
                        + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof))
                    print ERROR_STR + "Failed to verify consistency for " + self.name
                elif new["sha256_root_hash"] != str(base64.b64encode(res[1])):
                    self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \
                        + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof))
                    print ERROR_STR + "Failed to verify consistency for " + self.name

        except Exception, e:
            self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:"  + str(e))
            print ERROR_STR + "Could not verify consistency for " + self.url


# def verify_inclusion_all(old, new):
#     for url in old:
#         try:
#             if old[url] and new[url]:
#                 if old[url]["tree_size"]!= new[url]["tree_size"]:
#                     entries = []

#                     while len(entries) + old[url]["tree_size"]!= new[url]["tree_size"]:
#                         entries += get_entries(url, str(int(old[url]["tree_size"]) + len(entries)), new[url]["tree_size"] -1)["entries"]
#                         print "Got " + str(len(entries)) + " entries..."

#                     success = True
#                     for i in entries:
#                         h = get_leaf_hash(base64.b64decode(i["leaf_input"]))
#                         if not verify_inclusion_by_hash(url, h):
#                             success = False

#                     if success:
#                         print time.strftime("%H:%M:%S") + " Verifying inclusion for " + str(len(entries)) + " new entries in " + url + " ...OK"
#                     else:
#                         print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url
#                         errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url)
#         except:
#             print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url
#             errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url)



# def verify_inclusion_by_hash(base_url, leaf_hash):
#     try: 
#         tmp_sth = get_sth(base_url)
#         proof = get_proof_by_hash(base_url, leaf_hash, tmp_sth["tree_size"])

#         decoded_inclusion_proof = []
#         for item in proof["audit_path"]:
#             decoded_inclusion_proof.append(base64.b64decode(item))
        
#         root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, proof["leaf_index"], tmp_sth["tree_size"], leaf_hash))

#         if tmp_sth["sha256_root_hash"] == root:
#             return True
#         else:
#             print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(proof["leaf_index"]) + " in " + base_url
#             return False
#     except:
#         print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for hashed entry in " + base_url
#         return False

# def verify_inclusion_by_index(base_url, index):
#     try: 
#         tmp_sth = get_sth(base_url)
#         proof = get_proof_by_index(base_url, index, tmp_sth["tree_size"])

#         decoded_inclusion_proof = []
#         for item in proof["audit_path"]:
#             decoded_inclusion_proof.append(base64.b64decode(item))

#         root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, index, tmp_sth["tree_size"], get_leaf_hash(base64.b64decode(proof["leaf_input"]))))

#         if tmp_sth["sha256_root_hash"] == root:
#             print time.strftime('%H:%M:%S') + " Verifying inclusion for entry " + str(index) + " in " + base_url + "...OK."
#         else:
#             print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url
#             errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url)
#     except:
#         print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url
#         errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url)

# def get_proof_by_index(baseurl, index, tree_size):
#     try:
#         params = urllib.urlencode({"leaf_index":index,
#                                    "tree_size":tree_size})
#         result = \
#           urlopen(baseurl + "ct/v1/get-entry-and-proof?" + params).read()
#         return json.loads(result)
#     except urllib2.HTTPError, e:
#         print "ERROR:", e.read()
#         sys.exit(0)


def setup_domain_monitoring():
    monitored_domains = []
    try:
        with open(CONFIG.DOMAINS_FILE) as fp:
            for line in fp:
                tmp = json.loads(line)
                for domain in tmp:
                    if domain["url"] in CONFIG.MONITORED_DOMAINS:
                        md = monitored_domain(domain["url"])
                        md.load_entries(domain["entries"])
                        monitored_domains.append(md)
    except IOError:
        pass

    for md in CONFIG.MONITORED_DOMAINS: 
        tmp = monitored_domain(md)
        if not tmp in monitored_domains:
            print "New domain (not in file) " + md
            tmp.set()   
            monitored_domains.append(tmp)
    return monitored_domains


def main(args):
    monitored_domains = setup_domain_monitoring()

    # Create logs
    logs = []
    try:
        # Create log objects
        for item in CONFIG.CTLOGS:
            logs.append(ctlog(item["name"], item["url"], item["key"], item["id"], item["build"]))
        print time.strftime('%H:%M:%S') + " Setting up monitor for " + str(len(logs)) + " logs..."

        # Set up state 
        for log in logs:
            if os.path.isfile(log.savefile):
                log.load()

        # Build new entries 
        for log in logs:
            log.incremental_build()

        # Main loop: Monitor
        print time.strftime('%H:%M:%S') + " Running... (see logfiles for output)"
        while True:
            time.sleep(CONFIG.INTERVAL)
            for log in logs:
                log.update_roots()
                old_sth = log.sth
                log.update_sth() # Should this be done if later checks fail? (reorder?)
                if old_sth["timestamp"] != log.sth["timestamp"]:
                    log.verify_progress(old_sth)
                    log.verify_consistency(old_sth)
                    log.incremental_build()

            for md in monitored_domains:
                md.update()

    # Normal exit of the program
    except KeyboardInterrupt:
        print time.strftime('%H:%M:%S') + ' Received interrupt from user. Saving and exiting....'
        for log in logs:
            log.save()

        # Save info about monitored domains
        domain_dict = []
        for md in monitored_domains:
            domain_dict.append(md.to_dict())
        open(CONFIG.DOMAINS_FILE, 'w').write(json.dumps(domain_dict))

    # Something went horribly wrong!
    except Exception, err:
        print Exception, err
        for log in logs:
            log.save()

        # Save info about monitored domains
        domain_dict = []
        if len(monitored_domains) > 0:
            for md in monitored_domains:
                domain_dict.append(md.to_dict())
            open(CONFIG.DOMAINS_FILE, 'w').write(json.dumps(domain_dict))


if __name__ == '__main__':
    if CONFIG.OUTPUT_DIR and not os.path.exists(CONFIG.OUTPUT_DIR):
        os.makedirs(CONFIG.OUTPUT_DIR)

    if CONFIG.DB_PATH and not os.path.exists(CONFIG.DB_PATH):
        os.makedirs(CONFIG.DB_PATH)

    main(args)