diff options
Diffstat (limited to 'monitor/josef_mover.py')
| -rwxr-xr-x | monitor/josef_mover.py | 26 |
1 files changed, 15 insertions, 11 deletions
diff --git a/monitor/josef_mover.py b/monitor/josef_mover.py index a985f65..2267d00 100755 --- a/monitor/josef_mover.py +++ b/monitor/josef_mover.py @@ -128,13 +128,14 @@ def move_entry(first, last, source, dest): def check_overlap(source, dests): - PROCESS_COUNT = 100 + PROCESS_COUNT = 50 for log in source: print "Checking overlap from " + log["name"] sth = get_sth(log["url"]) first = 0 - last = int(sth["tree_size"]) + # last = int(sth["tree_size"]) + last = 23000 # split into tasks processes = [] @@ -155,6 +156,7 @@ def check_overlap(source, dests): args=(tmp_start, tmp_last, [log], dests, filename)) p.start() processes.append(p) + time.sleep(1.3) for p in processes: p.join() @@ -192,8 +194,8 @@ def check_submission_inner(d_log, item, inclusions): inclusions.append(d_log["name"]) def check_submission_outer(first, last, s_log, dest, logfile=None): - MAX_CHUNK_SIZE = 50 - MAX_RETRIES = 2 + MAX_CHUNK_SIZE = 65 + MAX_RETRIES = 3 idx = 0 while first + idx < last: if first + idx + MAX_CHUNK_SIZE < last: @@ -204,15 +206,16 @@ def check_submission_outer(first, last, s_log, dest, logfile=None): retries = 0 while retries <= MAX_RETRIES: try: - # print "Getting " + str(first + idx) + " to " + str(tmp_last) + print "Getting " + str(first + idx) + " to " + str(tmp_last) entries = get_entries(s_log["url"], first + idx, tmp_last)["entries"] break except Exception, e: time.sleep(1) - # if retries == MAX_RETRIES: - # print "FAILED!", Exception, e - # else: - # print "Failed, retrying...(" + str(retries) + ")" + if retries == MAX_RETRIES: + print "ERROR, could not download entries " + str(first + idx) + " to " + str(tmp_last), Exception, e + raise Exception("Failed to get entries") + else: + print "Failed, retrying...(" + str(retries) + ") ", Exception, e retries += 1 @@ -242,7 +245,7 @@ def check_submission_outer(first, last, s_log, dest, logfile=None): idx += len(entries) def check_stress_submission(first, last, source, dest, filename=None): - MAX_RETRIES = 2 + MAX_RETRIES = 4 for s_log in source: retries = 0 @@ -258,6 +261,7 @@ def check_stress_submission(first, last, source, dest, filename=None): print "FAILED!", Exception, e else: print "Failed, retrying...(" + str(retries) + ")" + pass retries += 1 @@ -292,7 +296,7 @@ def log(fn, string): if __name__ == "__main__": - source = [CTLOGS[2]] + source = [CTLOGS[4]] dests = CTLOGS # stress_test([CTLOGS[0]]) |