From d412c2d31c17ffd11d653899e2d64ae4059bb6aa Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Sun, 8 Feb 2015 01:53:40 +0100
Subject: Wait after first submission. Continue on http error 400. Print
 submission rate and number of submissions every 1000 submissions.

---
 tools/certtools.py  |  4 +++-
 tools/submitcert.py | 39 +++++++++++++++++++++++++++++++++------
 2 files changed, 36 insertions(+), 7 deletions(-)

(limited to 'tools')

diff --git a/tools/certtools.py b/tools/certtools.py
index af94fb8..e1ca57a 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -117,7 +117,9 @@ def add_chain(baseurl, submission):
             json.dumps(submission)).read()
         return json.loads(result)
     except urllib2.HTTPError, e:
-        print "ERROR:", e.read()
+        print "ERROR", e.code,":", e.read()
+        if e.code == 400:
+            return None
         sys.exit(1)
     except ValueError, e:
         print "==== FAILED REQUEST ===="
diff --git a/tools/submitcert.py b/tools/submitcert.py
index 2bb0c1a..1b87b53 100755
--- a/tools/submitcert.py
+++ b/tools/submitcert.py
@@ -14,6 +14,7 @@ import itertools
 from certtools import *
 import os
 import signal
+import select
 
 from multiprocessing import Pool
 
@@ -36,22 +37,28 @@ def submitcert(certfile):
     try:
         result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)})
     except SystemExit:
+        print "EXIT:", certfile
+        select.select([], [], [], 1.0)
         return None
 
     timing_point(timing, "addchain")
 
+    if result == None:
+        print "ERROR for certfile", certfile
+        return timing["deltatimes"]
+
     try:
         if check_sig:
             check_sct_signature(baseurl, certs[0], result)
             timing_point(timing, "checksig")
     except AssertionError, e:
-        print "ERROR:", e
+        print "ERROR:", certfile, e
         return None
     except urllib2.HTTPError, e:
-        print "ERROR:", e
+        print "ERROR:", certfile, e
         return None
     except ecdsa.keys.BadSignatureError, e:
-        print "ERROR: bad signature"
+        print "ERROR: bad signature", certfile
         return None
 
     if lookup_in_log:
@@ -98,15 +105,35 @@ def submitcert(certfile):
     timing_point(timing, "lookup")
     return timing["deltatimes"]
 
-p = Pool(1, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
+p = Pool(16, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
+
+nsubmitted = 0
+lastprinted = 0
+starttime = datetime.datetime.now()
+
+print len(certfiles), "certs"
+
+submitcert(certfiles[0])
+nsubmitted += 1
+select.select([], [], [], 3.0)
 
 try:
-    for timing in p.imap_unordered(submitcert, certfiles):
+    for timing in p.imap_unordered(submitcert, certfiles[1:]):
         if timing == None:
+            print "error"
+            print "submitted", nsubmitted
             p.terminate()
             p.join()
             sys.exit(1)
-        print timing
+        nsubmitted += 1
+        deltatime = datetime.datetime.now() - starttime
+        deltatime_f = deltatime.seconds + deltatime.microseconds / 1000000.0
+        rate = nsubmitted / deltatime_f
+        if nsubmitted > lastprinted + len(certfiles) / 10:
+            print nsubmitted, "rate %.1f" % rate
+            lastprinted = nsubmitted
+        #print timing, "rate %.1f" % rate
+    print "submitted", nsubmitted
 except KeyboardInterrupt:
     p.terminate()
     p.join()
-- 
cgit v1.1