diff options
-rw-r--r-- | packaging/docker/base-debian:jessie/Dockerfile | 2 | ||||
-rw-r--r-- | packaging/docker/catlfish-dev/Dockerfile | 27 | ||||
-rwxr-xr-x | packaging/docker/catlfish-dev/merge.sh (renamed from packaging/docker/catlfish-merge/merge.sh) | 2 | ||||
-rwxr-xr-x | packaging/docker/catlfish-dev/start.sh | 22 | ||||
-rw-r--r-- | packaging/docker/catlfish-dev/supervisord.conf | 2 | ||||
-rw-r--r-- | packaging/docker/catlfish-merge/Dockerfile | 41 | ||||
-rw-r--r-- | packaging/docker/catlfish-merge/supervisord.conf | 6 | ||||
-rw-r--r-- | reltool.config | 2 | ||||
-rw-r--r-- | test/catlfish-test-local-merge.cfg | 1 | ||||
-rwxr-xr-x | tools/merge.py | 15 | ||||
-rwxr-xr-x | verifycert.erl | 6 |
11 files changed, 60 insertions, 66 deletions
diff --git a/packaging/docker/base-debian:jessie/Dockerfile b/packaging/docker/base-debian:jessie/Dockerfile index dbd5d37..864c239 100644 --- a/packaging/docker/base-debian:jessie/Dockerfile +++ b/packaging/docker/base-debian:jessie/Dockerfile @@ -2,5 +2,3 @@ FROM debian:jessie RUN apt-get update RUN echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections RUN apt-get -y -q upgrade -RUN apt-get -y -q install supervisor -RUN mkdir -p /var/log/supervisor diff --git a/packaging/docker/catlfish-dev/Dockerfile b/packaging/docker/catlfish-dev/Dockerfile index 11ebed5..241c58d 100644 --- a/packaging/docker/catlfish-dev/Dockerfile +++ b/packaging/docker/catlfish-dev/Dockerfile @@ -2,18 +2,29 @@ # /usr/local/etc/catlfish/catlfish.config so mounting # /usr/local/etc/catlfish is recommended. This can be done using the # `-v' flag to `docker run'. - +# # NOTE: The directory on the host system that's mounted at # /var/local/db/catlfish in the container has to be writable by a host # user with uid 147. - +# +# Example, running a frontend node: +# $ docker run -v /etc/catlfish:/usr/local/etc/catlfish:ro catlfish +# frontend /usr/local/catlfish +# +# Example, running a merge node: # $ docker run -v /etc/catlfish:/usr/local/etc/catlfish:ro catlfish +# merge /usr/local/catlfish /var/local/db/catlfish-merge FROM erlang RUN apt-get update RUN echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections + +# For building. RUN apt-get -y -q install gcc git make curl +# For merge. +RUN apt-get -y -q install git python-ecdsa python-yaml + # Build dependencies in /usr/local/src. WORKDIR /usr/local/src @@ -41,10 +52,8 @@ RUN make -C plop RUN git clone https://git.nordu.net/catlfish.git RUN make -C catlfish PREFIX=/usr/local all release -# Config dir and database dir are mounted from host using `-v' to -# 'docker run'. +# Config dir is mounted from host using `-v' to 'docker run'. VOLUME /usr/local/etc/catlfish -VOLUME /var/local/db/catlfish # Create a catlfish user. RUN groupadd --gid 147 catlfish @@ -54,12 +63,12 @@ RUN useradd --uid 147 --gid 147 catlfish # /var/run/catlfish and not in /usr/local/etc/catlfish, so symlink. RUN mkdir /var/run/catlfish WORKDIR /var/run/catlfish -RUN mkdir erlang_log sasl_log supervisord_log +RUN mkdir erlang_log sasl_log merge_log RUN chown -R catlfish:catlfish /var/run/catlfish RUN ln -s /usr/local/etc/catlfish/catlfish.config /var/run/catlfish/ -# Run supervisord. -ADD supervisord.conf /etc/supervisor/ +ADD merge.sh /usr/local/catlfish/ +ADD start.sh /var/run/catlfish/ WORKDIR /var/run/catlfish USER catlfish -CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] +ENTRYPOINT ["/var/run/catlfish/start.sh"] diff --git a/packaging/docker/catlfish-merge/merge.sh b/packaging/docker/catlfish-dev/merge.sh index b3658cd..304d8a4 100755 --- a/packaging/docker/catlfish-merge/merge.sh +++ b/packaging/docker/catlfish-dev/merge.sh @@ -12,6 +12,8 @@ DBDIR="$3" [ -d $DBDIR/chains ] || mkdir $DBDIR/chains [ -e $DBDIR/logorder ] || touch $DBDIR/logorder +echo "merge: ERL_LIBS: $ERL_LIBS" + date echo "merge: waiting $(expr $S1 / 60)m$(expr $S1 % 60)s before merging for the first time" sleep $S1 diff --git a/packaging/docker/catlfish-dev/start.sh b/packaging/docker/catlfish-dev/start.sh new file mode 100755 index 0000000..8b3697d --- /dev/null +++ b/packaging/docker/catlfish-dev/start.sh @@ -0,0 +1,22 @@ +#! /bin/sh + +role=$1; [ -n "$1" ] && shift +erlbase=$1; [ -n "$1" ] && shift +database=$1; [ -n "$1" ] && shift + +case $role in + frontend|storage|signing) + $erlbase/bin/run_erl \ + /var/run/catlfish/ \ + /var/run/catlfish/erlang_log/ \ + "exec $erlbase/bin/erl -config catlfish" + ;; + merge) + ERL_LIBS=$erlbase/lib/catlfish-0.6.0-dev.ez/catlfish-0.6.0-dev + ERL_LIBS=$ERL_LIBS:$erlbase/lib/lager-2.1.1.ez/lager-2.1.1 + export ERL_LIBS + $erlbase/merge.sh 60 3600 $database > merge_log/stdout 2> merge_log/stderr + ;; + *) + echo "catlfish: unknown role: $role" +esac diff --git a/packaging/docker/catlfish-dev/supervisord.conf b/packaging/docker/catlfish-dev/supervisord.conf index f3493e5..8eda9b9 100644 --- a/packaging/docker/catlfish-dev/supervisord.conf +++ b/packaging/docker/catlfish-dev/supervisord.conf @@ -3,4 +3,4 @@ nodaemon=true [program:catlfish] childlogdir=/var/run/catlfish/supervisord_log -command=/usr/local/catlfish/bin/run_erl /var/run/catlfish/ /var/run/catlfish/erlang_log/ "exec /usr/local/catlfish/bin/erl -config catlfish" +command=/usr/local/catlfish/start.sh diff --git a/packaging/docker/catlfish-merge/Dockerfile b/packaging/docker/catlfish-merge/Dockerfile deleted file mode 100644 index 9a82d5f..0000000 --- a/packaging/docker/catlfish-merge/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Docker file for catlfish merge. -# -# NOTE: The directory on the host system that's mounted at -# /var/local/db/catlfish-merge in the container has to be writable by -# a host user with uid 147. -# -# $ docker run \ -# -v /etc/catlfish:/usr/local/etc/catlfish:ro \ -# -v /var/local/db/catlfish-merge:/var/local/db/catlfish-merge \ -# catlfish-merge - -FROM erlang -RUN apt-get update -RUN echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections -RUN apt-get -y -q install git python-ecdsa python-yaml - -WORKDIR /usr/local/src -RUN git clone https://git.nordu.net/catlfish.git - -# Config dir and database dir are mounted from host using `-v' to -# 'docker run'. -VOLUME /usr/local/etc/catlfish -VOLUME /var/local/db/catlfish-merge - -# Copy merge.sh. -RUN mkdir -p /usr/local/catlfish/bin -ADD merge.sh /usr/local/catlfish/bin/ - -# Create a catlfish user. -RUN groupadd --gid 147 catlfish -RUN useradd --uid 147 --gid 147 catlfish - -# We run from /var/run/catlfish. -RUN mkdir -p /var/run/catlfish/supervisord_log -RUN chown -R catlfish:catlfish /var/run/catlfish -WORKDIR /var/run/catlfish - -# Run supervisord. -ADD supervisord.conf /etc/supervisor/ -USER catlfish -CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/packaging/docker/catlfish-merge/supervisord.conf b/packaging/docker/catlfish-merge/supervisord.conf deleted file mode 100644 index deaf308..0000000 --- a/packaging/docker/catlfish-merge/supervisord.conf +++ /dev/null @@ -1,6 +0,0 @@ -[supervisord] -nodaemon=true - -[program:catlfish-merge] -childlogdir=/var/run/catlfish -command=/usr/local/catlfish/bin/merge.sh 300 1200 /var/local/db/catlfish-merge diff --git a/reltool.config b/reltool.config index 31fd1b0..c40be85 100644 --- a/reltool.config +++ b/reltool.config @@ -2,7 +2,7 @@ {sys, [ {erts, [{mod_cond, derived}, {app_file, strip}]}, {app_file, strip}, - {rel, "catlfish", "0.2.0-dev", + {rel, "catlfish", "0.6.0-dev", [ kernel, stdlib, diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg index c4dffbb..766c872 100644 --- a/test/catlfish-test-local-merge.cfg +++ b/test/catlfish-test-local-merge.cfg @@ -7,3 +7,4 @@ paths: logpublickey: tests/keys/logkey.pem privatekeys: tests/privatekeys verifycert_bin: ../verifycert.erl + known_roots: tests/known_roots/ diff --git a/tools/merge.py b/tools/merge.py index 0435b77..76ffede 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -226,7 +226,16 @@ def verify_entry(verifycert, entry, hash): mtl = unpacked[0] assert hash == get_leaf_hash(mtl) s = struct.pack(">I", len(entry)) + entry - verifycert.stdin.write(s) + try: + verifycert.stdin.write(s) + except IOError, e: + sys.stderr.write("merge: unable to write to verifycert process: ") + while 1: + line = verifycert.stdout.readline() + if line: + sys.stderr.write(line) + else: + sys.exit(1) result_length_packed = verifycert.stdout.read(4) (result_length,) = struct.unpack(">I", result_length_packed) result = verifycert.stdout.read(result_length) @@ -251,8 +260,8 @@ for hash in new_entries: entries_to_fetch[storagenode["name"]].append(hash) break -verifycert = subprocess.Popen(paths["verifycert_bin"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE) +verifycert = subprocess.Popen([paths["verifycert_bin"], paths["known_roots"]], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) added_entries = 0 for storagenode in storagenodes: diff --git a/verifycert.erl b/verifycert.erl index b9a3753..d364adf 100755 --- a/verifycert.erl +++ b/verifycert.erl @@ -1,6 +1,6 @@ #!/usr/bin/env escript %% -*- erlang -*- -%%! -pa ebin -pa lib/catlfish-0.6.0-dev.ez/catlfish-0.6.0-dev/ebin -pa lib/lager-2.1.1.ez/lager-2.1.1/ebin +%%! -pa lib/catlfish-0.6.0-dev.ez/catlfish-0.6.0-dev/ebin -pa lib/lager-2.1.1.ez/lager-2.1.1/ebin write_reply(Bin) -> Length = size(Bin), @@ -36,6 +36,6 @@ loop(RootCerts) -> loop(RootCerts) end. -main(_) -> - Certs = x509:read_pemfiles_from_dir("tests/known_roots/"), +main([KnownRoots]) -> + Certs = x509:read_pemfiles_from_dir(KnownRoots), loop(Certs). |