From 49b6e85963ef55fb6cfa1876fe825730f95658bc Mon Sep 17 00:00:00 2001 From: Linus Nordberg Date: Fri, 25 Nov 2016 00:47:13 +0100 Subject: Parallelise merge_fetch.py. NOTE: Not supporting permdb yet! We're still not passing the tests because merge_backup.py exits when the secondary merge disappears. --- tools/merge | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'tools/merge') diff --git a/tools/merge b/tools/merge index 0d1bf0e..4ba0438 100755 --- a/tools/merge +++ b/tools/merge @@ -5,7 +5,7 @@ import os import sys import signal from time import sleep -from mergetools import parse_args +from mergetools import parse_args, terminate_child_procs from multiprocessing import Process import merge_fetch, merge_backup, merge_sth, merge_dist @@ -21,27 +21,38 @@ def run_once(): return ret def term(signal, arg): + terminate_child_procs() sys.exit(1) def run_continuously(pidfilepath): """Run continuously.""" - parts = ('fetch', merge_fetch), ('backup', merge_backup), ('sth', merge_sth), ('dist', merge_dist) + parts = (('fetch', merge_fetch), + ('backup', merge_backup), + ('sth', merge_sth), + ('dist', merge_dist)) procs = {} for part, mod in parts: procs[part] = Process(target=mod.main, name='merge_%s' % part) - procs[part].daemon = True procs[part].start() + #print >>sys.stderr, "DEBUG:", part, "started, pid", procs[part].pid if pidfilepath: open(pidfilepath, 'w').write(str(os.getpid()) + '\n') signal.signal(signal.SIGTERM, term) - while True: + retval = 0 + keep_going = True + while keep_going: + sleep(1) for name, p in procs.items(): if not p.is_alive(): print >>sys.stderr, "\nERROR:", name, "process is gone; exiting" - return 1 - sleep(1) + retval = 1 # Fail. + keep_going = False + break + + terminate_child_procs() + return retval def main(): """Main""" -- cgit v1.1