reading log size history from log files

1 files changed, 62 insertions, 3 deletions

diff --git a/monitor/josef_logreader.py b/monitor/josef_logreader.py
index 2b3a473..868ab00 100755
--- a/monitor/josef_logreader.py
+++ b/monitor/josef_logreader.py
@@ -99,11 +99,21 @@ def print_errors(l):
 
 def print_overlap(l):
     log = read_loglog(l)
+    source = log[0].split("[")[0].split()[-1]
+    print "\nOverlaps in " + source + ":"
     entry_count = {}
+    metadata = {"unique":0, "valid":0}
     total = 0
     for line in log:
         # line = item[TIME_LEN:]
         logs = json.loads(line.split("logs: ")[-1][:-1].replace("'", '"'))
+        if not source in logs: # don't check entries that were submitted after measurement start
+            break
+        if len(logs) == 1:
+            metadata["unique"] += 1
+        if "pilot" in logs or "rocketeer" in logs or "aviator" in logs:
+            metadata["valid"] += 1
+
         for l in logs:
             if l in entry_count:
                 entry_count[l] += 1
@@ -113,7 +123,54 @@ def print_overlap(l):
     
     # print entry_count
     for e in entry_count:
-        print e + ", " + str(entry_count[e]) + " (" + str(int(100 * float(entry_count[e])/float(total))) + "%)"
+        if e != "plausible":
+            print e + ", " + str(entry_count[e]) + " (" + str(int(100 * float(entry_count[e])/float(total))) + "%)"
+
+    print ""
+    for e in metadata:
+        print e + ", " + str(metadata[e]) + " (" + str(int(100 * float(metadata[e])/float(total))) + "%)"
+
+def write_history_data(l):
+    log = read_loglog(l)
+    data = {}
+    start_date = datetime.date(2015, 9, 29)
+    end_date = datetime.date(2015, 10, 20)
+
+    # initialize with all dates
+    for single_date in daterange(start_date, end_date):
+        data[single_date.strftime("%Y-%m-%d")] = 0
+
+    # parse log data
+    for line in log:
+        date = line[:TIME_LEN -10]
+        try:
+            size = line.split("Size: ")[1].split(",")[0]
+            data[date] = size
+        except:
+            pass
+
+    # put in list and sort
+    data_list = []
+    for item in data:
+        data_list.append((item, data[item]))
+    data_list.sort(key=lambda tup: tup[0])
+
+    # fill out missing data points and print
+    prev_val = 0
+    for item in data_list:
+        if item[1] > prev_val:
+            val = item[1]
+        else:
+            val = prev_val
+        prev_val = val
+        print item[0], val
+
+
+
+def daterange(start_date, end_date):
+    for n in range(int ((end_date - start_date).days)):
+        yield start_date + datetime.timedelta(n)
+
 
 if __name__ == "__main__":
     OVERLAP_STR = "_overlap.log"
@@ -122,13 +179,15 @@ if __name__ == "__main__":
         if log == "monitor.log":
             pass
         elif log[- len(OVERLAP_STR):] == OVERLAP_STR:
-            print_overlap(log)
-        # else:
+            # print_overlap(log)
+            pass
+        else:
         #     print log
         #     print_log_stats(log)
         #     print_average_age(log)
         #     print_errors(log)
         #     print ""
+            write_history_data(log)