adding code for analyzing overlap

5 files changed, 107 insertions, 22 deletions

diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index cef06a6..fa553a0 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -4,10 +4,10 @@
 import sys
 import os
 from josef_lib import *
-import leveldb
+# import leveldb
 import argparse
 import json
-from josef_leveldb import *
+# from josef_leveldb import *
 from datetime import datetime as dt
 # from josef_monitor import verify_inclusion_by_hash
 from monitor_conf import *
@@ -69,6 +69,12 @@ def update_roots(log):
 
 
 if __name__ == '__main__':
+    for log in CTLOGS:
+        url = log["url"]
+        try:
+            get_entries(url,2001,2001)
+        except Exception, e:
+            print "Failed to get entry from " + log["name"], e
 
     # dbdir = "tmpdb/"
     # entry = get_entries(ctlogs[0]["url"], 1,1)["entries"]
diff --git a/monitor/josef_lib.py b/monitor/josef_lib.py
index db9dad3..e7b1e05 100644
--- a/monitor/josef_lib.py
+++ b/monitor/josef_lib.py
@@ -23,6 +23,12 @@ from Crypto.Hash import SHA256
 import Crypto.PublicKey.RSA as RSA
 from Crypto.Signature import PKCS1_v1_5
 
+def time_str(ts = None):
+    if ts is None:
+        return datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
+    else:
+        return datetime.datetime.fromtimestamp(ts / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
+
 def compare_lists(new, old):
     added_items  = []
     removed_items  = []
@@ -342,7 +348,7 @@ def add_chain(baseurl, submission):
         result = urlopen(baseurl + "ct/v1/add-chain", json.dumps(submission)).read()
         return json.loads(result)
     except urllib2.HTTPError, e:
-        print "ERROR", e.code,":", e.read()
+        # print "ERROR", e.code,":", e.read()
         # if e.code == 400:
         return None
         # sys.exit(1)
@@ -360,7 +366,7 @@ def add_prechain(baseurl, submission):
             json.dumps(submission)).read()
         return json.loads(result)
     except urllib2.HTTPError, e:
-        print "ERROR", e.code,":", e.read()
+        # print "ERROR", e.code,":", e.read()
         # if e.code == 400:
         return None
         # sys.exit(1)
diff --git a/monitor/josef_logreader.py b/monitor/josef_logreader.py
index 5f6f768..2b3a473 100755
--- a/monitor/josef_logreader.py
+++ b/monitor/josef_logreader.py
@@ -5,6 +5,7 @@ import sys
 import time
 import datetime
 import os
+import json
 
 from monitor_conf import *
 
@@ -96,18 +97,38 @@ def print_errors(l):
             break # comment this line to print all errors ever
             pass
 
+def print_overlap(l):
+    log = read_loglog(l)
+    entry_count = {}
+    total = 0
+    for line in log:
+        # line = item[TIME_LEN:]
+        logs = json.loads(line.split("logs: ")[-1][:-1].replace("'", '"'))
+        for l in logs:
+            if l in entry_count:
+                entry_count[l] += 1
+            else:
+                entry_count[l] = 1
+        total += 1
+    
+    # print entry_count
+    for e in entry_count:
+        print e + ", " + str(entry_count[e]) + " (" + str(int(100 * float(entry_count[e])/float(total))) + "%)"
 
 if __name__ == "__main__":
+    OVERLAP_STR = "_overlap.log"
     logs = get_logs()
     for log in logs:
         if log == "monitor.log":
             pass
-        else:
-            print log
-            print_log_stats(log)
-            print_average_age(log)
-            print_errors(log)
-            print ""
+        elif log[- len(OVERLAP_STR):] == OVERLAP_STR:
+            print_overlap(log)
+        # else:
+        #     print log
+        #     print_log_stats(log)
+        #     print_average_age(log)
+        #     print_errors(log)
+        #     print ""
 
 
 
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index b8ebd52..e371c86 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -344,11 +344,7 @@ class ctlog:
 #         print "ERROR:", e.read()
 #         sys.exit(0)
 
-def time_str(ts = None):
-    if ts is None:
-        return datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
-    else:
-        return datetime.datetime.fromtimestamp(ts / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
+
 
 def setup_domain_monitoring():
     monitored_domains = []
diff --git a/monitor/josef_mover.py b/monitor/josef_mover.py
index 08ec05d..ac79448 100755
--- a/monitor/josef_mover.py
+++ b/monitor/josef_mover.py
@@ -50,10 +50,10 @@ def print_reply(rep, entry):
 def is_new_timestamp(ts):
     MAX_TIMEDIFF = 300 # 5 min, allows for some clock skew
     ts_time = datetime.datetime.fromtimestamp(ts / 1000, UTC()).strftime('%Y-%m-%d %H:%M:%S')
-    start_time = datetime.datetime.utcnow().strftime('2013-10-19 00:00:00')
-    delta_time = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') - datetime.datetime.strptime(ts_time, '%Y-%m-%d %H:%M:%S')
+    start_time = datetime.datetime.utcnow().strftime('2015-10-19 00:00:00')
+    # delta_time = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') - datetime.datetime.strptime(ts_time, '%Y-%m-%d %H:%M:%S')
     # print delta_time.seconds
-    if delta_time.seconds > MAX_TIMEDIFF:
+    if ts_time < start_time:
         return False
     else:
         return True
@@ -123,17 +123,73 @@ def move_entry(first, last, source, dest):
                     print "FAILED!\n"
             print s_log["name"] + "[" + str(first + i) + "] found in " + str(len(inclusions)) + " logs: ", inclusions
 
+def check_inclusion_by_submission(first, last, source, dest):
+    # print entries
+    for s_log in source:
+        try:
+            entries = get_entries(s_log["url"], first, last)["entries"]
+        except:
+            log("Failed to get entries from " + s_log["name"])
+
+        # print "\n\nSource: " + s_log["name"] + "\n"
+        for i in range(len(entries)):
+        # for item in entries:
+            item = entries[i]
+            inclusions = []
+            for d_log in dests:
+                # print "Log: " + d_log["name"]
+                try:
+                    entry = extract_original_entry(item)
+                    if entry[2]:
+                        precert = True
+                    else:
+                        precert = False
+                    submission = []
+
+                    for e in entry[0]:
+                        submission.append(base64.b64encode(e))
+
+                    if entry[2]:
+                        res = add_prechain(d_log["url"], {"chain" : submission})
+                    else:
+                        res = add_chain(d_log["url"], {"chain" : submission})
+                    # print_reply(res, entry)
+
+                    if not is_new_timestamp(res["timestamp"]):
+                        inclusions.append(d_log["name"])
+
+                    # time.sleep(5)
+                except KeyboardInterrupt:
+                    sys.exit()
+                except:
+                    # print "FAILED!\n"
+                    pass
+            s = s_log["name"] + "[" + str(first + i) + "] found in " + str(len(inclusions)) + " logs: " + str(inclusions)
+            print s
+            logfile = OUTPUT_DIR + s_log["name"] + "_overlap.log"
+            log(logfile, s)
+
+def log(fn, string):
+    logfile = fn
+    s = time_str() + " " + string
+    with open(logfile, 'a') as f:
+        f.write(s + "\n")
+        f.close()
 
 if __name__ == "__main__":
-    source = [CTLOGS[0]]
+    source = [CTLOGS[9]]
     dests = CTLOGS
     # source = ctlogs
     # dests = ctlogs
+    for tmp_log in source:
+        sth = get_sth(tmp_log["url"])
+
 
-    first = 100
-    last = 102
+        first = 0
+        last = int(sth["tree_size"]) - 1
+        # print last
 
-    move_entry(first, last, source,dests)
+        check_inclusion_by_submission(first, last, [tmp_log], dests)
     # check_inclusion_all(first,last,source, dests)