adding gaol

7 files changed, 372 insertions, 3 deletions

diff --git a/monitor/gaol/gaol.ct.nordu.net.pem b/monitor/gaol/gaol.ct.nordu.net.pem
new file mode 100644
index 0000000..c75c001
--- /dev/null
+++ b/monitor/gaol/gaol.ct.nordu.net.pem
@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEE0E7UyZA2XkTJC2Rlcx0DbXkI1Y0
++2OgoJQ0O+pwtVRCiVGHdU08i4m8MKx2r8FWbpHVgt6V0rLS8rYBErfSVA==
+-----END PUBLIC KEY-----
diff --git a/monitor/gaol/gaol_auditor.py b/monitor/gaol/gaol_auditor.py
new file mode 100755
index 0000000..d2da59b
--- /dev/null
+++ b/monitor/gaol/gaol_auditor.py
@@ -0,0 +1,220 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-  
+
+
+import time
+import datetime
+import base64
+import argparse
+import errno
+import subprocess
+
+import json
+
+from gaol_lib import *
+from lib import *
+import os.path
+
+
+parser = argparse.ArgumentParser(description="")
+parser.add_argument('--config', default="gaol_conf.py")
+args = parser.parse_args()
+
+# Import from config file
+if os.path.isfile(args.config):
+    modules = map(__import__, [args.config[:-2]])
+    CONFIG = modules[0]
+    ERROR_STR = CONFIG.ERROR_STR
+else:
+    print "Config file not found!"
+    ERROR_STR = "(local)ERROR: "
+    sys.exit()
+
+def email(s):
+    for addr in CONFIG.EMAIL_ADDR:
+        p = subprocess.Popen(
+        ["mail", "-s", CONFIG.EMAIL_SUBJECT, addr],
+        stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    p.communicate(s)
+
+
+class ctlog:
+    def __init__(self, name, url, key, log_id=None, build=True):
+        self.name = name
+        self.url = url
+        self.key = key
+        self.log_id = log_id
+        self.logfile = CONFIG.OUTPUT_DIR + name + ".log"
+        self.savefile = CONFIG.OUTPUT_DIR + name + "-state-info.json"
+        self.subtree = [[]]
+        # self.fe_ips = {}
+        self.sth = None
+        self.entries = 0
+        self.root_hash = None
+        self.build = build
+
+        self.saved_sth = None
+        self.saved_entries = None
+        self.saved_subtree = None
+
+        self.log("Starting monitor")
+
+
+
+    def to_dict(self):
+        d = {}
+        d["sth"] = self.sth
+        return d
+
+    def save(self):
+        self.log("Saving state to file")
+        open(self.savefile, 'w').write(json.dumps(self.to_dict()))
+
+    def load(self):
+        self.log("Loading state from file")
+        try:
+            f = open(self.savefile)
+            s = f.read()
+            d = json.loads(s)
+            self.sth = d["sth"]
+
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                return None
+            raise e
+
+
+    def log(self, string):
+        s = time_str() + " " + string
+        with open(self.logfile, 'a') as f:
+            f.write(s + "\n")
+            f.close()
+        email(s)
+
+    def update_sth(self):
+        try:
+            new_sth = get_sth(self.url)
+        except Exception, e:
+            self.log(ERROR_STR + "Failed to fetch STH. " +str(e))
+            return
+
+        try:
+            check_sth_signature(self.url, new_sth, base64.b64decode(self.key))
+        except:
+            self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth))
+
+        if self.sth:
+            sth_time = time_str(new_sth["timestamp"])
+            if new_sth["timestamp"] != self.sth["timestamp"]:
+                self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time)
+                self.sth = new_sth
+        else:
+            self.log("Setting initial STH: " + str(new_sth))
+            self.sth = new_sth
+
+    
+    def verify_progress(self, old):
+        new = self.sth
+        try:
+            if new["tree_size"] == old["tree_size"]:
+                if old["sha256_root_hash"] != new["sha256_root_hash"]:
+                    self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new))
+            elif new["tree_size"] < old["tree_size"]:
+                self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new))
+
+            if new["timestamp"] < old["timestamp"]:
+                self.log(ERROR_STR + "Regression in timestamps! Old:" + str(old) + " New:" + str(new))
+            else:
+                age = time.time() - new["timestamp"]/1000
+                sth_time = time_str(new["timestamp"])
+                roothash = new['sha256_root_hash']
+                if age > 24 * 3600:
+                    s = ERROR_STR + "STH is older than 24h: %s UTC" % (sth_time)
+                    self.log(s + str(new))
+                    print s
+                elif age > 12 * 3600:
+                    s = "WARNING: STH is older than 12h: %s UTC" % (sth_time)
+                    self.log(s)
+                elif age > 6 * 3600:
+                    s = "WARNING: STH is older than 6h: %s UTC" % (sth_time)
+                    self.log(s)
+        except Exception, e:
+            self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e))
+
+    def verify_consistency(self, old):
+        new = self.sth
+        try:
+            if old["tree_size"]!= new["tree_size"]:
+                consistency_proof = get_consistency_proof(self.url, old["tree_size"], new["tree_size"])
+                decoded_consistency_proof = []
+                for item in consistency_proof:
+                    decoded_consistency_proof.append(base64.b64decode(item))
+                res = verify_consistency_proof(decoded_consistency_proof, old["tree_size"], new["tree_size"], base64.b64decode(old["sha256_root_hash"]))
+                
+                if old["sha256_root_hash"] != str(base64.b64encode(res[0])):
+                    self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \
+                        + str(old) + " New:" + str(new) + " Calculated:" + str(base64.b64encode(res[0]))\
+                        + " Proof:" + str(consistency_proof))
+                elif new["sha256_root_hash"] != str(base64.b64encode(res[1])):
+                    self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \
+                        + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof))
+
+        except Exception, e:
+            self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:"  + str(e))
+
+
+
+def main(args):
+    # Create logs
+    logs = []
+    try:
+        # Create log objects
+        for item in CONFIG.CTLOGS:
+            logs.append(ctlog(item["name"], item["url"], item["key"], item["id"], item["build"]))
+        print time_str() + " Setting up monitor for " + str(len(logs)) + " logs..."
+
+        # Set up state 
+        for log in logs:
+            if os.path.isfile(log.savefile):
+                log.load()
+
+
+        # Main loop: Auditor
+        print time_str() + " Running... (see logfiles for output)"
+        while True:
+            for log in logs:
+                old_sth = log.sth
+
+                log.update_sth()
+                if old_sth and old_sth["timestamp"] != log.sth["timestamp"]:
+                    log.verify_progress(old_sth)        
+                    log.verify_consistency(old_sth)      # Does rollback on critical fail
+                    pass
+            time.sleep(CONFIG.INTERVAL)
+
+    # Normal exit of the program
+    except KeyboardInterrupt:
+        print time_str() + ' Received interrupt from user. Saving and exiting....'
+        for log in logs:
+            log.save()
+
+    # Something went horribly wrong!
+    except Exception, err:
+        print Exception, err
+        for log in logs:
+            log.save()
+
+
+
+if __name__ == '__main__':
+    if CONFIG.OUTPUT_DIR and not os.path.exists(CONFIG.OUTPUT_DIR):
+        os.makedirs(CONFIG.OUTPUT_DIR)
+
+    main(args)
+
+
+
+
+
+
diff --git a/monitor/gaol/gaol_conf.py b/monitor/gaol/gaol_conf.py
new file mode 100644
index 0000000..7019989
--- /dev/null
+++ b/monitor/gaol/gaol_conf.py
@@ -0,0 +1,29 @@
+ # All configuration for the CT monitor is done from this file!
+
+# interval (in seconds) between updates
+INTERVAL = 60 
+
+# Directories for various output files
+OUTPUT_DIR = "output/"
+
+# Some strings
+ERROR_STR = "ERROR: "
+EMAIL_SUBJECT = "GAOL"
+
+# Email addresses for error messages 
+EMAIL_ADDR = [
+"josef@nordu.net",
+]
+
+# CT logs and associated keys
+CTLOGS = [
+    {"name" : "gaol",
+    "url" : "https://gaol.ct.nordu.net/open/",
+    "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEE0E7UyZA2XkTJC2Rlcx0DbXkI1Y0+2OgoJQ0O+pwtVRCiVGHdU08i4m8MKx2r8FWbpHVgt6V0rLS8rYBErfSVA==",
+    "id" : "5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=",
+    "build" : False},
+]
+
+
+
+
diff --git a/monitor/gaol/gaol_lib.py b/monitor/gaol/gaol_lib.py
new file mode 100755
index 0000000..07e674b
--- /dev/null
+++ b/monitor/gaol/gaol_lib.py
@@ -0,0 +1,66 @@
+import json
+import urllib
+import urllib2
+import ssl
+import base64
+
+from lib import *
+
+class sslparameters:
+    sslcontext = None
+
+def get_opener():
+    try:
+        opener = urllib2.build_opener(urllib2.HTTPSHandler(context=sslparameters.sslcontext))
+    except TypeError:
+        opener = urllib2.build_opener(urllib2.HTTPSHandler())
+    return opener
+
+def urlopen(url, data=None):
+    return get_opener().open(url, data)
+
+def get_sth(baseurl):
+    result = urlopen(baseurl + "gaol/v1/get-sth").read()
+    return json.loads(result)
+
+def get_entries(baseurl, start, end):
+    params = urllib.urlencode({"start":start, "end":end})
+    # try:
+    result = urlopen(baseurl + "gaol/v1/get-entries?" + params).read()
+    return json.loads(result)
+
+def get_consistency_proof(baseurl, tree_size1, tree_size2):
+    # try:
+        params = urllib.urlencode({"first":tree_size1,
+                                   "second":tree_size2})
+        result = \
+          urlopen(baseurl + "gaol/v1/get-sth-consistency?" + params).read()
+        return json.loads(result)["consistency"]
+    # except urllib2.HTTPError, e:
+    #     print "ERROR:", e.read()
+        # sys.exit(1)
+
+def extract_original_entry(entry):
+    leaf_input =  base64.decodestring(entry["leaf_input"])
+    (data_blob, timestamp, issuer_key_hash) = unpack_mtl(leaf_input)
+    return (data_blob, timestamp)
+
+def make_blob(data):
+	return base64.b64encode(data)
+
+def add_blob(baseurl, blob):
+    try:
+        result = urlopen(baseurl + "gaol/v1/add-blob", json.dumps({"blob" : blob})).read()
+        return json.loads(result)
+    except urllib2.HTTPError, e:
+        return "ERROR " + str(e.code) + " : " + e.read()
+        # if e.code == 400:
+        return None
+        # sys.exit(1)
+    except ValueError, e:
+        print "==== FAILED REQUEST ===="
+        print submission
+        print "======= RESPONSE ======="
+        print result
+        print "========================"
+        raise e
\ No newline at end of file
diff --git a/monitor/gaol/gaol_sct.txt b/monitor/gaol/gaol_sct.txt
new file mode 100644
index 0000000..569a7b2
--- /dev/null
+++ b/monitor/gaol/gaol_sct.txt
@@ -0,0 +1,17 @@
+{u'timestamp': 1448874793847, u'signature': u'BAMASDBGAiEAhlobK0j+9Ya8ssxJ7Fbhn0MP9gXW1YihNKRyhkwGak0CIQCLpQ6bIRI+7l0E7umLCPIkW0e79S3g5jYo9iNNVBSrzA==', u'sct_version': 0, u'id': u'5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=', u'extensions': u''}
+
+{u'timestamp': 1448876147795, u'signature': u'BAMARjBEAiB83TwGbzXwgoE0xR3/LYlADdvEexqciFne1Rh68B2wWQIgGoROCsKWHGrRjfeOVHzPutJlUQ3ahfKfMO2ffa+zLuc=', u'sct_version': 0, u'id': u'5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=', u'extensions': u''}
+
+{"sth": {"timestamp": 1448639930117, "sha256_root_hash": "FliDqU/S5o+qMPS42nwYy9b1/kKEldeedad/5vDl+UA=", "tree_size": 2, "tree_head_signature": "BAMARzBFAiBEOSbEdnzBGeUV+AGzJUs/ekf5Sq/SRbrxxiDKN23yNAIhAOTrwhYFHg3a6iLrGoz0QoIjOW8PckRJ/1oayj3993Cl"}}
+
+{"sth": {"timestamp": 1448878465713, "sha256_root_hash": "uaFzynY1Y6csM1ZeJfL73C+pqTt3NOI2kMXwEZxe7B0=", "tree_size": 4, "tree_head_signature": "BAMARzBFAiEAiUfFg4YCjjAb5bkJz0FfqQUYT/noY+6Y1b+hjvbySGoCIH0ItuAWS5LPQjhAKTXAJtu+ODxXNQmgcUu/bNfhV01r"}}
+
+{"sth" : {"timestamp":1448893523957,"sha256_root_hash":"APf+TzOLNooNAF0Ka59C9n+dLRccMKZ/S5TNKULuolM=","tree_size":5,"tree_head_signature":"BAMARjBEAiAmmxUE3xC1HePGEZvUaubi0WlzwhW+SiTu5hvvxquLhQIgAoRms8ogotonyuoFHZaxWxJMkh9d9l+ZsJ/Jee2VTTo="}}
+
+{u'timestamp': 1448886901898, u'signature': u'BAMARTBDAh9nairFQko6KHZGpS+X4PmCwTUmUP1MpAOSphmA+Zf+AiBW5Ok+y1aDCNqKfmRyaY/sBSMWDI8NNdE56ouRvR07BQ==', u'sct_version': 0, u'id': u'5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=', u'extensions': u''}
+
+
+{"sth": {"timestamp": 1447820379698, "sha256_root_hash": "adJNwPSYP0XOadNQEVF1nZSVZ+ojN9ORPXEdjRMhC8U=", "tree_size": 9867417, "tree_head_signature": "BAMARzBFAiEAwa1SAOMb5k/QQTDQvTPqDkGi43esqg04Em1MHJ6tpckCIBedSUATxgSB3AfIxq/UlP5KtlfuaMGRZYcbUIF0UWyC"}}
+
+{u'timestamp': 1448895405459, u'signature': u'BAMARzBFAiEA5BtisYNDhFY3GmNCi+KPrXhi2yHc4uETnthF6QGJ44UCICDYBNrcwgSjKt2BSVFn7Jn8G9StLo3NrsXLUKYQZFoz', u'sct_version': 0, u'id': u'5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=', u'extensions': u''}
+
diff --git a/monitor/gaol/gaol_test.py b/monitor/gaol/gaol_test.py
new file mode 100755
index 0000000..dbcadc4
--- /dev/null
+++ b/monitor/gaol/gaol_test.py
@@ -0,0 +1,33 @@
+#!/usr/bin/python
+
+import base64
+
+from gaol_conf import *
+from gaol_lib import *
+
+def test_get_entry(url, idx):
+    print "Testing to fetch an entry..."
+    entry = get_entries(url,idx,idx)["entries"][0]
+    print "Received: " + extract_original_entry(entry)[0]
+
+def test_submission(url, data):
+    print "\nTesting to submitt a sample text..."
+    blob = make_blob(data)
+    res = add_blob(url,blob)
+    print res
+
+def test_consistency_proof(url, idx1, idx2):
+    print "\nTesing a consistency proof"
+    res = get_consistency_proof(url, idx1, idx2)
+    print res
+
+if __name__ == '__main__':
+    url = CTLOGS[0]["url"]
+    test_get_entry(url,1)
+    test_get_entry(url,2)
+    test_get_entry(url,3)
+    test_get_entry(url,4)
+    test_get_entry(url,5)
+    # test_get_entry(url,6)
+    # test_submission(url, "Progress (n.): The process through which the Internet has evolved from smart people in front of dumb terminals to dumb people in front of smart terminals.")
+    # test_consistency_proof(url, 2, 3)
\ No newline at end of file
diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index 5939b82..b3acf1c 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -175,11 +175,11 @@ def email(s):
         # command = 'echo "' + s + '" | mail -s "' + EMAIL_SUBJECT + '" ' + addr
         # os.system("bash -c '" + command + "'")
         p = subprocess.Popen(
-        ["mail", "-s", '"' + EMAIL_SUBJECT + '"', addr],
+        ["mail", "-s", EMAIL_SUBJECT, addr],
         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
         stderr=subprocess.PIPE)
-    parsed = p.communicate(s)
-    print parsed
+    p.communicate(s)
+    # print parsed
 
 
 if __name__ == '__main__':