Wait for merge in tests

7 files changed, 96 insertions, 26 deletions

diff --git a/test/scripts/light-system-test-run-1.sh b/test/scripts/light-system-test-run-1.sh
index 1ad56a8..585a829 100755
--- a/test/scripts/light-system-test-run-1.sh
+++ b/test/scripts/light-system-test-run-1.sh
@@ -15,7 +15,7 @@ for certfile in ${top_srcdir}/tools/testcerts/cert[1-5].txt ${top_srcdir}/tools/
     python ${top_srcdir}/tools/submitcert.py --parallel=1 --store $certfile --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 done
 python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg  || fail "GC failed"
-do_merge
+do_merge 7
 check_sth
 
 assert_equal "Tree size" "$(get_treesize)" 7
@@ -36,10 +36,8 @@ ${top_srcdir}/tools/to_catlfish.py to_erl nodes/merge-2/ "init:stop()"
 python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert6.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 echo NOTE: merge backup should fail with 111 Connection refused
 
-${top_srcdir}/tools/merge_fetch.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
-${top_srcdir}/tools/merge_backup.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg && fail "Merge backup should have failed"
-${top_srcdir}/tools/merge_sth.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
-${top_srcdir}/tools/merge_dist.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
+do_merge 7
+sleep 3
 
 assert_equal "Tree size" "$(get_treesize)" 7
 
@@ -50,7 +48,7 @@ for i in 1 2 3 4 5 6 7 8 9 10; do
     sleep 0.5 ; \
     if curl -s --cacert httpsca/demoCA/cacert.pem -4 https://localhost:8181 > /dev/null ; then break; fi
 done
-do_merge
+do_merge 8
 
 assert_equal "Tree size" "$(get_treesize)" 8
 
diff --git a/test/scripts/light-system-test-run-3.sh b/test/scripts/light-system-test-run-3.sh
index bb042d2..6071001 100755
--- a/test/scripts/light-system-test-run-3.sh
+++ b/test/scripts/light-system-test-run-3.sh
@@ -7,7 +7,8 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 . ${top_srcdir}/test/scripts/testutils.sh
 
-do_merge
+do_merge 8
+sleep 3
 check_sth
 
 assert_equal "Tree size" "$(get_treesize)" 8
@@ -16,7 +17,7 @@ python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tool
 
 assert_equal "Tree size" "$(get_treesize)" 8
 
-do_merge
+do_merge 9
 
 assert_equal "Tree size" "$(get_treesize)" 9
 
diff --git a/test/scripts/light-system-test-run-4.sh b/test/scripts/light-system-test-run-4.sh
index a2490f9..51e1fc5 100755
--- a/test/scripts/light-system-test-run-4.sh
+++ b/test/scripts/light-system-test-run-4.sh
@@ -7,7 +7,8 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 . ${top_srcdir}/test/scripts/testutils.sh
 
-do_merge
+do_merge 9
+sleep 3
 check_sth
 
 assert_equal "Tree size" "$(get_treesize)" 9
diff --git a/test/scripts/light-system-test-run-5.sh b/test/scripts/light-system-test-run-5.sh
index 355e179..cf0bfaf 100755
--- a/test/scripts/light-system-test-run-5.sh
+++ b/test/scripts/light-system-test-run-5.sh
@@ -14,7 +14,8 @@ for machine in ${MACHINES}; do \
 done
 
 
-do_merge
+do_merge 9
+sleep 3
 check_sth
 
 assert_equal "Tree size" "$(get_treesize)" 9
diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh
index 731b6e5..d3a9672 100644
--- a/test/scripts/testutils.sh
+++ b/test/scripts/testutils.sh
@@ -36,8 +36,16 @@ check_sth_host() {
 }
 
 do_merge() {
-    ${top_srcdir}/tools/merge --config machine/merge/catlfish-test.cfg --timing --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
-    sleep 5                     # FIXME: Just wait for dist instead.
+    n=15
+    expected_treesize=$1
+    while [ $n -gt 0 ]; do
+        ${top_srcdir}/tools/merge --config machine/merge/catlfish-test.cfg --timing --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
+        ${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+        n=$((n-1))
+        treesize=$(get_treesize)
+        [ "${treesize}" == "${expected_treesize}" ] && return
+        sleep 1
+    done
 }
 
 init_log() {
diff --git a/tools/mergestatus.py b/tools/mergestatus.py
new file mode 100755
index 0000000..80d755e
--- /dev/null
+++ b/tools/mergestatus.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017, NORDUnet A/S.
+# See LICENSE for licensing information.
+#
+import sys
+import json
+import urllib2
+import time
+import requests
+import base64
+from base64 import b64encode
+from mergetools import parse_args, get_nfetched, hexencode, hexdecode, \
+     get_logorder, get_sth
+from certtools import create_ssl_context, get_public_key_from_file, \
+     timing_point, create_sth_signature, write_file, check_sth_signature, \
+     build_merkle_tree
+
+def main():
+    args, config, localconfig = parse_args()
+    paths = localconfig["paths"]
+    mergenodes = config.get("mergenodes", [])
+    mergedb = paths["mergedb"]
+    sthfile = mergedb + "/sth"
+    currentsizefile = mergedb + "/fetched"
+
+    sth = json.loads(open(sthfile, "r").read())
+    currentsize = json.loads(open(currentsizefile, "r").read())
+    
+    print >>sys.stderr, currentsize["index"]+1,
+    
+    for mergenode in mergenodes:
+        if mergenode["name"] == config["primarymergenode"]:
+            continue
+        verifiedfile = mergedb + "/verified." + mergenode["name"]
+        try:
+            tree = json.loads(open(verifiedfile, "r").read())
+            print >>sys.stderr, tree["tree_size"],
+        except (IOError, ValueError):
+            pass
+    print >>sys.stderr, sth["tree_size"],
+    print
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/tools/testcase1.py b/tools/testcase1.py
index 5192074..12079c8 100755
--- a/tools/testcase1.py
+++ b/tools/testcase1.py
@@ -148,13 +148,28 @@ def get_and_check_entry(timestamp, chain, leaf_index, baseurl):
                     len(submittedcertchain),
                     len(submittedcertchain))
 
-def merge():
-    rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg",
-                            "--localconfig", testdir + "/catlfish-test-local-merge.cfg"])
-    sleep(5)                      # FIXME: Just wait for dist instead.
-    return rv
-
-mergeresult = merge()
+def correct_tree_size(expected):
+    for baseurl in baseurls:
+        sth = get_sth(baseurl)
+        tree_size = sth["tree_size"]
+        if tree_size != expected:
+            return False
+    return True
+
+def merge(expected=None, wait=0):
+    for i in range(10):
+        rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg",
+                                "--localconfig", testdir + "/catlfish-test-local-merge.cfg"])
+        if rv:
+            return rv
+        if i < wait:
+            sleep(1)
+            continue
+        if correct_tree_size(expected):
+            return 0
+    return 0
+
+mergeresult = merge(expected=0, wait=3)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
 
 for baseurl in baseurls:
@@ -164,7 +179,7 @@ testgroup("cert1")
 
 result1 = do_add_chain(cc1, baseurls[0])
 
-mergeresult = merge()
+mergeresult = merge(expected=1)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
 
 size_sth = {}
@@ -177,7 +192,7 @@ result2 = do_add_chain(cc1, baseurls[0])
 
 assert_equal(result2["timestamp"], result1["timestamp"], "timestamp")
 
-mergeresult = merge()
+mergeresult = merge(expected=1, wait=3)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
 
 for baseurl in baseurls:
@@ -195,7 +210,7 @@ testgroup("cert2")
 
 result3 = do_add_chain(cc2, baseurls[0])
 
-mergeresult = merge()
+mergeresult = merge(expected=2)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
 
 for baseurl in baseurls:
@@ -209,7 +224,7 @@ testgroup("cert3")
 
 result4 = do_add_chain(cc3, baseurls[0])
 
-mergeresult = merge()
+mergeresult = merge(expected=3)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
 
 for baseurl in baseurls:
@@ -224,7 +239,7 @@ testgroup("cert4")
 
 result5 = do_add_chain(cc4, baseurls[0])
 
-mergeresult = merge()
+mergeresult = merge(expected=4)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
 
 for baseurl in baseurls:
@@ -240,7 +255,7 @@ testgroup("cert5")
 
 result6 = do_add_chain(cc5, baseurls[0])
 
-mergeresult = merge()
+mergeresult = merge(expected=5)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
 
 for baseurl in baseurls:
@@ -253,7 +268,7 @@ get_and_validate_proof(result4["timestamp"], cc3, 2, 3, baseurls[0])
 get_and_validate_proof(result5["timestamp"], cc4, 3, 3, baseurls[0])
 get_and_validate_proof(result6["timestamp"], cc5, 4, 1, baseurls[0])
 
-mergeresult = merge()
+mergeresult = merge(expected=5, wait=3)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
 
 for first_size in range(1, 5):