diff options
| -rwxr-xr-x | tools/merge_fetch.py | 19 |
1 files changed, 9 insertions, 10 deletions
diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py index 10fd406..be9edce 100755 --- a/tools/merge_fetch.py +++ b/tools/merge_fetch.py @@ -127,17 +127,16 @@ def merge_fetch_worker(args, localconfig, storagenode, pipe): if to_fetch: logging.info("%s: fetching %d entries", storagenode["name"], len(to_fetch)) - fetchlist = list(to_fetch) with requests.sessions.Session() as session: - entries = get_entries(storagenode["name"], - "https://%s/" % storagenode["address"], - own_key, paths, fetchlist, - session=session) - for ehash in fetchlist: - entry = entries[ehash] - verify_entry(verifycert, entry, ehash) - pipe.send(('FETCHED', ehash, entry)) - to_fetch.remove(ehash) + for chunk in chunks(list(to_fetch), 100): + entries = get_entries(storagenode["name"], + "https://%s/" % storagenode["address"], + own_key, paths, chunk, session=session) + for ehash in chunk: + entry = entries[ehash] + verify_entry(verifycert, entry, ehash) + pipe.send(('FETCHED', ehash, entry)) + to_fetch.remove(ehash) new_entries = get_new_entries(storagenode["name"], "https://%s/" % storagenode["address"], |