diff options
author | Magnus Ahltorp <map@kth.se> | 2016-11-18 17:13:31 +0100 |
---|---|---|
committer | Magnus Ahltorp <map@kth.se> | 2016-11-18 17:13:31 +0100 |
commit | aa7be9990183d6e9a87037b88a1e33dca35304e3 (patch) | |
tree | bf496efbb11fe23d23597340a7ffe237d87b9fbb /tools/merge_fetch.py | |
parent | b48c1689e36bdcc65a34b4ab12763478b072a716 (diff) |
Use sessions in merge_fetch
Diffstat (limited to 'tools/merge_fetch.py')
-rwxr-xr-x | tools/merge_fetch.py | 30 |
1 files changed, 16 insertions, 14 deletions
diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py index 3028b30..ddd2f06 100755 --- a/tools/merge_fetch.py +++ b/tools/merge_fetch.py @@ -7,6 +7,7 @@ import sys import struct import subprocess +import requests from time import sleep from mergetools import get_logorder, verify_entry, get_new_entries, \ chunks, fsync_logorder, get_entries, add_to_logorder, \ @@ -63,20 +64,21 @@ def merge_fetch(args, config, localconfig): print >>sys.stderr, "getting %d entries from %s:" % \ (len(entries_to_fetch[storagenode["name"]]), storagenode["name"]), sys.stderr.flush() - for chunk in chunks(entries_to_fetch[storagenode["name"]], 100): - entries = get_entries(storagenode["name"], - "https://%s/" % storagenode["address"], - own_key, paths, chunk) - for ehash in chunk: - entry = entries[ehash] - verify_entry(verifycert, entry, ehash) - chainsdb.add(ehash, entry) - add_to_logorder(logorderfile, ehash) - logorder.append(ehash) - certsinlog.add(ehash) - added_entries += 1 - print >>sys.stderr, added_entries, - sys.stderr.flush() + with requests.sessions.Session() as session: + for chunk in chunks(entries_to_fetch[storagenode["name"]], 100): + entries = get_entries(storagenode["name"], + "https://%s/" % storagenode["address"], + own_key, paths, chunk, session=session) + for ehash in chunk: + entry = entries[ehash] + verify_entry(verifycert, entry, ehash) + chainsdb.add(ehash, entry) + add_to_logorder(logorderfile, ehash) + logorder.append(ehash) + certsinlog.add(ehash) + added_entries += 1 + print >>sys.stderr, added_entries, + sys.stderr.flush() print >>sys.stderr sys.stderr.flush() chainsdb.commit() |