Update from sunet/multiverse.ct-ops-2017-07-28-v01

13 files changed, 115 insertions, 63 deletions

diff --git a/.gitignore b/.gitignore
index e004c2d..0d20b64 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1 @@
 *.pyc
-**/*.pyc
diff --git a/addhost b/addhost
index 033c16b..1c34ce3 100755
--- a/addhost
+++ b/addhost
@@ -45,7 +45,7 @@ if [ ! -d $cmd_hostname ]; then
 fi
 
 if [ "$cmd_do_bootstrap" = "yes" ]; then
-   scp apt/cosmos_1.2-2_all.deb apt/bootstrap-cosmos.sh root@$cmd_hostname:
+   scp apt/cosmos_1.5-1_all.deb apt/bootstrap-cosmos.sh root@$cmd_hostname:
    ssh root@$cmd_hostname ./bootstrap-cosmos.sh $cmd_fqdn $rrepo $rtag
    ssh root@$cmd_hostname cosmos update 
    ssh root@$cmd_hostname cosmos apply
diff --git a/bump-tag b/bump-tag
index b163569..440809b 100755
--- a/bump-tag
+++ b/bump-tag
@@ -1,19 +1,30 @@
-#!/bin/sh
+#!/bin/bash
 
 set -e
 
 test -f cosmos.conf && . ./cosmos.conf
 
+echo "Fetching any updates from server:"
 git pull
+echo ""
 
-deftag=`basename $PWD`
+if [ "x$1" = "x" ]; then
+    deftag=`basename $PWD`
+else
+    deftag="$1"
+fi
 tagpfx=${tag:="$deftag"}
 
 last_tag=`git tag -l "${tagpfx}-*"|sort|tail -1`
 
-git tag -v $last_tag
+echo "Verifying last tag $last_tag:"
+(git tag -v $last_tag | grep ^gpg:) || true
+# again to not mask exit status of git with grep
+git tag -v $last_tag > /dev/null 2>&1
+echo ""
 
-PAGER=cat git diff $last_tag..master
+echo "Differences between tag $last_tag and what you are about to sign:"
+PAGER=cat git diff --color $last_tag..master
 
 iter=1
 ok=
@@ -29,8 +40,13 @@ while test -z "$ok"; do
     esac
 done
 
-echo using new tag $this_tag
-echo ONLY SIGN IF YOU APPROVE OF VERIFICATION AND DIFF ABOVE
+if [ "$deftag" != "$tagpfx" ]; then
+    echo -e "Using new tag \e[94m$this_tag\e[0m according to pattern in cosmos.conf"
+else
+    echo -e "Using new tag \e[94m$this_tag\e[0m"
+fi
+
+echo -e "\e[1mONLY SIGN IF YOU APPROVE OF VERIFICATION AND DIFF ABOVE\e[0m"
 
 # GITTAGEXTRA is for putting things like "-u 2117364A"
 
diff --git a/fabfile/__init__.py b/fabfile/__init__.py
index 3933104..8db5748 100644
--- a/fabfile/__init__.py
+++ b/fabfile/__init__.py
@@ -17,7 +17,7 @@ def all():
     env.hosts = cosmos_db()['members']['all']
 
 def cosmos():
-    run("/usr/local/bin/run-cosmos -v");
+    run("/usr/local/bin/run-cosmos");
 
 def upgrade():
     run("apt-get -qq update && apt-get -y -q dist-upgrade");
diff --git a/global/overlay/etc/puppet/cosmos-modules.conf b/global/overlay/etc/puppet/cosmos-modules.conf
index e4dc597..991a570 100644
--- a/global/overlay/etc/puppet/cosmos-modules.conf
+++ b/global/overlay/etc/puppet/cosmos-modules.conf
@@ -1,21 +1,18 @@
+#
 # name source (puppetlabs fq name or git url) upgrade (yes/no) tag-pattern
 #
 # NOTE that Git packages MUST be tagged with signatures by someone
 # in the Cosmos trust list. That is why all the URLs point to forked
 # versions in the SUNET github organization.
 #
-concat          git://github.com/SUNET/puppetlabs-concat.git yes sunet-*
-stdlib          git://github.com/SUNET/puppetlabs-stdlib.git yes sunet-*
-cosmos          git://github.com/SUNET/puppet-cosmos.git yes sunet-*
-ufw             git://github.com/SUNET/puppet-module-ufw.git yes sunet-*
-apt             git://github.com/SUNET/puppetlabs-apt.git yes sunet-*
-vcsrepo         git://github.com/SUNET/puppetlabs-vcsrepo.git yes sunet-*
-xinetd          git://github.com/SUNET/puppetlabs-xinetd.git yes sunet-*
-hiera-gpg       git://github.com/SUNET/hiera-gpg.git yes sunet-*
-augeas 		git://github.com/SUNET/puppet-augeas.git yes sunet-*
-docker          git://github.com/SUNET/garethr-docker.git yes sunet-*
-sunet 		git://github.com/SUNET/puppet-sunet.git yes sunet-*
-
+concat          https://github.com/SUNET/puppetlabs-concat.git yes sunet-*
+stdlib          https://github.com/SUNET/puppetlabs-stdlib.git yes sunet-*
+cosmos          https://github.com/SUNET/puppet-cosmos.git yes sunet-*
+ufw             https://github.com/SUNET/puppet-module-ufw.git yes sunet_dev-*
+apt             https://github.com/SUNET/puppetlabs-apt.git yes sunet_dev-*
+vcsrepo         https://github.com/SUNET/puppetlabs-vcsrepo.git yes sunet-*
+xinetd          https://github.com/SUNET/puppetlabs-xinetd.git yes sunet-*
+hiera-gpg       https://github.com/SUNET/hiera-gpg.git yes sunet-*
 #
 # Alternate sources you might or might not want to use:
 #concat          puppetlabs/concat no
@@ -24,15 +21,16 @@ sunet 		git://github.com/SUNET/puppet-sunet.git yes sunet-*
 #apt             puppetlabs/apt no
 #vcsrepo         puppetlabs/vcsrepo no
 #xinetd          puppetlabs/xinetd no
-#cosmos          git://github.com/leifj/puppet-cosmos.git yes
-#python          git://github.com/SUNET/puppet-python.git yes sunet-*
-#erlang          git://github.com/SUNET/garethr-erlang.git yes sunet-*
-#rabbitmq        git://github.com/SUNET/puppetlabs-rabbitmq.git yes sunet_dev-*
-#pound           git://github.com/SUNET/puppet-pound.git yes sunet_dev-*
-#augeas          git://github.com/SUNET/puppet-augeas.git yes sunet-*
-#bastion         git://github.com/SUNET/puppet-bastion.git yes sunet-*
-#postgresql      git://github.com/SUNET/puppetlabs-postgresql.git yes sunet_dev-*
-#munin           git://github.com/SUNET/ssm-munin.git yes sunet-*
-#nagios          git://github.com/SUNET/puppet-nagios.git yes sunet-*
-#staging         git://github.com/SUNET/puppet-staging.git yes sunet-*
-#apparmor        git://github.com/SUNET/puppet-apparmor.git yes sunet-*
+#cosmos          https://github.com/SUNET/puppet-cosmos.git yes
+#python          https://github.com/SUNET/puppet-python.git yes sunet-*
+#erlang          https://github.com/SUNET/garethr-erlang.git yes sunet-*
+#rabbitmq        https://github.com/SUNET/puppetlabs-rabbitmq.git yes sunet_dev-*
+#pound           https://github.com/SUNET/puppet-pound.git yes sunet_dev-*
+#augeas          https://github.com/SUNET/puppet-augeas.git yes sunet-*
+#bastion         https://github.com/SUNET/puppet-bastion.git yes sunet-*
+#postgresql      https://github.com/SUNET/puppetlabs-postgresql.git yes sunet_dev-*
+#munin           https://github.com/SUNET/ssm-munin.git yes sunet-*
+#nagios          https://github.com/SUNET/puppet-nagios.git yes sunet-*
+#staging         https://github.com/SUNET/puppet-staging.git yes sunet-*
+#apparmor        https://github.com/SUNET/puppet-apparmor.git yes sunet-*
+#docker          https://github.com/SUNET/garethr-docker.git yes sunet_dev-*
diff --git a/global/overlay/etc/puppet/puppet.conf b/global/overlay/etc/puppet/puppet.conf
index 1f834e8..88871f0 100644
--- a/global/overlay/etc/puppet/puppet.conf
+++ b/global/overlay/etc/puppet/puppet.conf
@@ -4,6 +4,7 @@ vardir=/var/lib/puppet
 ssldir=/var/lib/puppet/ssl
 rundir=/var/run/puppet
 factpath=$vardir/lib/facter
+templatedir=$confdir/templates
 node_terminus = exec
 external_nodes = /etc/puppet/cosmos_enc.py
 basemodulepath = /etc/puppet/modules:/etc/puppet/cosmos-modules:/usr/share/puppet/modules
diff --git a/global/overlay/usr/local/bin/run-cosmos b/global/overlay/usr/local/bin/run-cosmos
index a37d49f..5f2cbc1 100755
--- a/global/overlay/usr/local/bin/run-cosmos
+++ b/global/overlay/usr/local/bin/run-cosmos
@@ -1,22 +1,46 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Simplify running cosmos, with serialization if flock is available.
 #
 
-set -e
+readonly PROGNAME=$(basename "$0")
+readonly LOCKFILE_DIR=/tmp
+readonly LOCK_FD=200
 
-FLOCK=`which flock`
+lock() {
+    local prefix=$1
+    local fd=${2:-$LOCK_FD}
+    local lock_file=$LOCKFILE_DIR/$prefix.lock
 
-if [ -x "$FLOCK" ]; then
-    ($FLOCK --exclusive --wait 60 9 || exit 1
-	cosmos $* update
-	cosmos $* apply
-    )9>/var/lock/run-cosmos
-else
-    cosmos $* update
-    cosmos $* apply
-fi
+    # create lock file
+    eval "exec $fd>$lock_file"
+
+    # acquier the lock
+    flock -n $fd \
+        && return 0 \
+        || return 1
+}
+
+eexit() {
+    local error_str="$@"
+
+    echo $error_str
+    exit 1
+}
 
-touch /var/run/last-cosmos-ok.stamp
+main () {
+   lock $PROGNAME || eexit "Only one instance of $PROGNAME can run at one time."
+   cosmos $* update
+   cosmos $* apply
 
-find /var/lib/puppet/reports/ -type f -mtime +10 | xargs rm -f
+   touch /var/run/last-cosmos-ok.stamp
+
+   find /var/lib/puppet/reports/ -type f -mtime +10 | xargs rm -f
+}
+
+main $*
+
+if [ -f /cosmos-reboot ]; then
+   rm -f /cosmos-reboot
+   reboot
+fi
diff --git a/global/overlay/usr/local/sbin/cosmos_vm b/global/overlay/usr/local/sbin/cosmos_vm
index bf27576..5eec8f7 100755
--- a/global/overlay/usr/local/sbin/cosmos_vm
+++ b/global/overlay/usr/local/sbin/cosmos_vm
@@ -9,8 +9,8 @@ hostname="default"
 bridge="br0"
 cpus="1"
 mem="1024"
-repo="git://code.mnt.se/mnt-cosmos.git"
-tag="eduid-cosmos"
+repo="https://yourhost/myproj-cosmos.git"
+tag="cosmos-ops"
 ip=""
 gateway=""
 netmask=""
@@ -63,7 +63,7 @@ ssh_authorized_keys:
 runcmd:
    - ["mkdir","/tmp/seed"]
    - ["mount","/dev/vdb","/tmp/seed"]
-   - ["cp","/tmp/seed/bootstrap-cosmos.sh","/tmp/seed/cosmos_1.2-2_all.deb","/root"]
+   - ["cp","/tmp/seed/bootstrap-cosmos.sh","/tmp/seed/cosmos_1.5-1_all.deb","/root"]
    - ["cd","/root"]
    - "cd /root && /root/bootstrap-cosmos.sh ${hostname} ${repo} ${tag}"
 
@@ -112,7 +112,7 @@ fi
 
 mcopy -i ${seed} ${user_data} ::user-data 2>/dev/null
 mcopy -i ${seed} ${meta_data} ::meta-data 2>/dev/null
-mcopy -i ${seed} /etc/cosmos/apt/bootstrap-cosmos.sh /etc/cosmos/apt/cosmos_1.2-2_all.deb ::
+mcopy -i ${seed} /etc/cosmos/apt/bootstrap-cosmos.sh /etc/cosmos/apt/cosmos_1.5-1_all.deb ::
 mv ${seed} /var/lib/libvirt/images/
 virsh pool-refresh default
 
diff --git a/global/post-tasks.d/015cosmos-trust b/global/post-tasks.d/015cosmos-trust
index 447d875..74835e0 100755
--- a/global/post-tasks.d/015cosmos-trust
+++ b/global/post-tasks.d/015cosmos-trust
@@ -4,11 +4,19 @@ if [ -z "$COSMOS_KEYS" ]; then
    COSMOS_KEYS=/etc/cosmos/keys
 fi
 
+# Install new keys discovered in the $COSMOS_KEYS directory
 for k in $COSMOS_KEYS/*.pub; do
-   fp=`cosmos gpg --with-colons --with-fingerprint < $k| awk -F: '$1 == "pub" {print $5}'`
-   cosmos gpg --with-colons --fingerprint | grep -q ":$fp:" || cosmos gpg --import < $k
+   fp=`cosmos gpg --with-colons --with-fingerprint < $k | awk -F: '$1 == "pub" {print $5}'`
+   fp_in_db=`cosmos gpg --with-colons --fingerprint | grep ":$fp:"`
+   if [ "x`echo $fp_in_db | grep '^pub:e:'`" != "x" ]; then
+       echo "$0: Key expired, will re-import it from $k"
+       cosmos gpg --fingerprint $fp
+   fi
+   # The removal of any ^pub:e: entrys means to ignore expired keys - thereby importing them again.
+   echo $fp_in_db | grep -v "^pub:e:" | grep -q ":$fp:" || cosmos gpg --import < $k
 done
 
+# Delete keys no longer present in $COSMOS_KEYS directory
 for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5}'`; do
    seen="no"
    for k in $COSMOS_KEYS/*.pub; do
diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages
index 3e2e26e..9370e10 100755
--- a/global/post-tasks.d/018packages
+++ b/global/post-tasks.d/018packages
@@ -24,8 +24,8 @@ if [ -f $CONFIG ]; then
   # First pass to clone any new modules, and update those marked for updating.
   grep -E -v "^#" $CONFIG | (
     while read module src update pattern; do
-      # We only support git:// urls atm
-      if [ "${src:0:6}" = "git://" ]; then
+      # We only support git:// urls and https:// urls atm
+      if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then
         if [ ! -d $CACHE_DIR/scm/$module ]; then
           git clone -q $src $CACHE_DIR/scm/$module
         elif [ -d $CACHE_DIR/scm/$module/.git ]; then
@@ -63,7 +63,7 @@ if [ -f $CONFIG ]; then
   grep -E -v "^#" $CONFIG | (
     while read module src update pattern; do
       # We only support git:// urls atm
-      if [ "${src:0:6}" = "git://" ]; then
+      if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then
         # Verify git tag
         cd $CACHE_DIR/scm/$module
         TAG=$(git tag -l "${pattern:-*}" | sort | tail -1)
diff --git a/global/post-tasks.d/030puppet b/global/post-tasks.d/030puppet
index 6742949..b94b9ff 100755
--- a/global/post-tasks.d/030puppet
+++ b/global/post-tasks.d/030puppet
@@ -1,13 +1,13 @@
 #!/bin/sh
 
 if [ "x$COSMOS_VERBOSE" = "xy" ]; then
-   args="--verbose"
+   args="--verbose --show_diff"
 else
    args="--logdest=syslog"
 fi
 
 if [ -f /usr/bin/puppet -a -d /etc/puppet/manifests ]; then
    for m in `find /etc/puppet/manifests -name \*.pp`; do
-      puppet apply $args < $m
+      puppet apply $args $m
    done
 fi
diff --git a/global/post-tasks.d/099autoremove b/global/post-tasks.d/099autoremove
index 2cc6996..74b0aa4 100755
--- a/global/post-tasks.d/099autoremove
+++ b/global/post-tasks.d/099autoremove
@@ -1,4 +1,6 @@
 #!/bin/sh
 
-apt-get -qq update
-apt-get -qq -y autoremove
+if (( $RANDOM % 20 == 0)); then
+    apt-get -qq update
+    apt-get -qq -y autoremove
+fi
diff --git a/global/pre-tasks.d/030puppet b/global/pre-tasks.d/030puppet
index cdc9989..ef08016 100755
--- a/global/pre-tasks.d/030puppet
+++ b/global/pre-tasks.d/030puppet
@@ -9,9 +9,13 @@ stamp="$COSMOS_BASE/stamps/puppet-tools-v01.stamp"
 
 if ! test -f $stamp -a -f /usr/bin/puppet; then
     codename=`lsb_release -c| awk '{print $2}'`
-    wget -c http://apt.puppetlabs.com/puppetlabs-release-${codename}.deb
-    dpkg -i puppetlabs-release-${codename}.deb
-    rm -f puppetlabs-release-${codename}.deb*
+    puppetdeb="$COSMOS_REPO/apt/puppetlabs-release-${codename}.deb"
+    if [ ! -f $puppetdeb ]; then
+	echo "$0: Puppet deb for release $codename not found in $COSMOS_REPO/apt/"
+	echo "    Get it from https://apt.puppetlabs.com/ and put it in the Cosmos repo."
+	exit 1
+    fi
+    dpkg -i $puppetdeb
     apt-get update
     apt-get -y install puppet-common