diff options
| -rw-r--r-- | cdr1.sunet.se/overlay/etc/network/interfaces | 52 | ||||
| -rw-r--r-- | fabfile/__init__.py | 2 | ||||
| -rw-r--r-- | global/overlay/etc/cosmos/keys/ft-505152DD.pub | 88 | ||||
| -rw-r--r-- | global/overlay/etc/cosmos/keys/swold-BD115DD6.pub | 65 | ||||
| -rw-r--r-- | global/overlay/etc/cron.d/cosmos | 2 | ||||
| -rwxr-xr-x | global/overlay/etc/cron.daily/scriptherder_cleanup | 10 | ||||
| -rw-r--r-- | global/overlay/etc/puppet/cosmos-modules.conf | 1 | ||||
| -rw-r--r-- | global/overlay/etc/puppet/facter/cosmos.rb | 22 | ||||
| -rw-r--r-- | global/overlay/etc/puppet/hiera.yaml | 14 | ||||
| -rw-r--r-- | global/overlay/etc/puppet/manifests/cosmos-site.pp | 150 | ||||
| -rw-r--r-- | global/overlay/etc/puppet/modules/sunet/manifests/server.pp | 47 | ||||
| -rw-r--r-- | global/overlay/etc/puppet/puppet.conf | 4 | ||||
| -rw-r--r-- | global/overlay/etc/scriptherder/check/cosmos.ini | 3 | ||||
| -rwxr-xr-x | global/overlay/usr/local/bin/scriptherder | 302 | ||||
| -rwxr-xr-x | global/post-tasks.d/015cosmos-trust | 7 |
15 files changed, 540 insertions, 229 deletions
diff --git a/cdr1.sunet.se/overlay/etc/network/interfaces b/cdr1.sunet.se/overlay/etc/network/interfaces index 7475c35..edcc8fd 100644 --- a/cdr1.sunet.se/overlay/etc/network/interfaces +++ b/cdr1.sunet.se/overlay/etc/network/interfaces @@ -22,40 +22,42 @@ auto lo iface lo inet loopback -auto eth0 -iface eth0 inet manual -bond-master bond0 -bond-primary eth0 +auto em1 +iface em1 inet manual + bond-master bond0 + bond-primary em1 -auto eth1 -iface eth1 inet manual -bond-master bond0 +auto em2 +iface em2 inet manual + bond-master bond0 # bond0 is the bonding NIC and can be used like any other normal NIC. auto bond0 -iface bond0 inet static - address 130.242.125.70 - netmask 255.255.255.192 - network 130.242.125.64 - broadcast 130.242.125.127 - gateway 130.242.125.65 - # dns-* options are implemented by the resolvconf package, if installed - dns-nameservers 130.242.80.14 130.242.80.99 - dns-search sunet.se - bond-mode active-backup - bond-miimon 100 - bond-slaves none - -iface bond0 inet6 static - address 2001:6b0:8:4::70 - netmask 64 - gateway 2001:6b0:8:4::1 +iface bond0 inet manual + bond-mode active-backup + bond-miimon 100 + bond-slaves none # Bridge interface for KVM guests auto br0 -iface br0 inet dhcp +iface br0 inet static pre-up sleep 4 # must make br0 start _after_ bond0 is configured bridge_ports bond0 bridge_stp off bridge_fd 0 bridge_maxwait 0 + # + address 130.242.125.70 + netmask 255.255.255.192 + network 130.242.125.64 + broadcast 130.242.125.127 + gateway 130.242.125.65 + # dns-* options are implemented by the resolvconf package, if installed + dns-nameservers 130.242.80.14 130.242.80.99 + dns-search sunet.se + +iface br0 inet6 static + pre-up sleep 4 # must make br0 start _after_ bond0 is configured + address 2001:6b0:8:4::70 + netmask 64 + gateway 2001:6b0:8:4::1 diff --git a/fabfile/__init__.py b/fabfile/__init__.py index d87fbdd..3933104 100644 --- a/fabfile/__init__.py +++ b/fabfile/__init__.py @@ -17,7 +17,7 @@ def all(): env.hosts = cosmos_db()['members']['all'] def cosmos(): - run("cosmos update ; cosmos -v apply"); + run("/usr/local/bin/run-cosmos -v"); def upgrade(): run("apt-get -qq update && apt-get -y -q dist-upgrade"); diff --git a/global/overlay/etc/cosmos/keys/ft-505152DD.pub b/global/overlay/etc/cosmos/keys/ft-505152DD.pub index 092b32e..433fd6d 100644 --- a/global/overlay/etc/cosmos/keys/ft-505152DD.pub +++ b/global/overlay/etc/cosmos/keys/ft-505152DD.pub @@ -1,5 +1,5 @@ -----BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.14 (GNU/Linux) +Version: GnuPG v1 mQENBFJB3u8BCADKh5OKbT0QJGM9e5ZNaQ1D5nICfM2u9S4x/iBsIsCt7W7h022j hkW0AKFEhqUixLzl9TgjAMawW3V+59C/FkGWdvyGtOLu90tnjLXhFFc/wG5kv7Fq @@ -8,13 +8,13 @@ s8CiC82VftOr/QWKNys+kTGMgv0rpBgZ0ZmNe4qKTBKrVlzESMD/bWOVga70Rfsp MRY0cKK/V6gUDvaGtivg06SJNJzup8XzLYIclgwU3+U7c7YlseH7sUQlMUJ+Q2ij c+P3EkLNU2IOrJ/m9QwHl2EYG0GDaKz5IQGZABEBAAG0I0ZyZWRyaWsgVGh1bGlu IDxmcmVkcmlrQHRodWxpbi5uZXQ+iQE+BBMBCgAoAhsDBgsJCAcDAgYVCAIJCgsE -FgIDAQIeAQIXgAUCUqc44gUJAVKn6wAKCRAZjBr9UFFS3SU6B/482umbN+mjTtZT -RV8kqwvEawr5MP92sF5FNfgYmo4MVmUbyY/i7jrXEXKR4SlQIfE0eIxBqAFXJVdU -ZF8tyNay85tZXuZ+pFbIPUMLadnq54HY6QrMXiFR45Q+fNNCluLqDB+p0ANmL4nO -kboJnUGsm/c87gLuUda1Eo4S0opvYkTy7OmSh4PMAEZxObjU82svaD4eoqLbllF0 -8vAz/KoCScMQ+YqUhvCAhkxE8oHlvPLfo3VYkF0Gy8ofaj3cE68KUdXhlfdRizMn -jXJDXuVNMmUojEM6u1j2tRObeUoITnPZRs7PID/wLnhG66WcF7CyGyjQbQX7VNck -VYn7hMaZiQEcBBABCgAGBQJSQ+AeAAoJEE7yk05OqkIyTswH/jmbf6BKVwhpYNZz +FgIDAQIeAQIXgAUCU4LqJwUJAi5ZNQAKCRAZjBr9UFFS3Q7UCACOApsR4fqVXcZF +49df+DxlGXLLzjiIL9dezduTcFqGAxFUCqC/hnw26UKmvUTshYvnUC336FCdg2u7 +oOxIE+u9+In3M8sqnNWax0lBG/Y3Gmk4YwJH1cL1phXwFDh4xU2xrGPS1hkVOtUO +l8GEB+v+l9P/WgMMPbdNQr/J4UnlZ7jZpeE7QadbvCE7q1U1Rw9wYG9o5c/BzQ/X +htQz/oedmZlyyftHppAL3jGjNWnSWf042hn9nFsPLpJjEXWy+XVo0pQOBfitt+Oj +sap1DEyuAFqEwbVIQyGCTyXIwo88VnLxZ6EkR9VvX3RnccVr03izqk1JoULkmLh9 +USHxrBtCiQEcBBABCgAGBQJSQ+AeAAoJEE7yk05OqkIyTswH/jmbf6BKVwhpYNZz S/nlKGIhf0R0g4LPPOep25EwJfmUk54Mvi48/TBdZ9cy5Kwj3nlTXvviC72c7KvI ViZuZwO+jv0JOr2H5+xtT2zfWOeYFK0vlHPk1dUI++h84bnM/DCyGWdP6My1leRN VRySQI9E4NfHI03EN8fhoSKwHXfwrZOu5O8/8t4odJjo9IhnRiPvqFDqfbKDby+X @@ -75,30 +75,50 @@ JpT4fGGkFAfltEZLvU2i7PfjVAFLYlNpHAo++7Ihpk/mtIYuNp2Jc3On1XGDCOY+ FTHNrkzF4zaU1eqJCzBC3UVu9oc28Ztsqf1OKr/hFRzL+uoxDnFYOspOtuIigDs5 lvXYbMbiIn6CyhvDhLXmODHvYk1O2qN9ZzVQZ+xTpJHSD3WUMNa0cPfln/2banqZ Njoa684QCdPx5v1ut3YXlNzsbvZzqWicFXOSncqWGBlF8AYZdisgF3oUWcZ5UU9m -m/xFOH2iKSgx95LT2KGPpt6Hrf99MeYQPrBDufyinsrnPqzwACaMqMxMiI6tuQEN -BFJB3u8BCADcdMuW3XLX+TBrZj2Kg9j7TOgRJZFSDSl8yDdzk3/nZKa41TjBnWf/ -gU6mqpPmZ5idSQ7DnFWg7bZOV/NF0Be6IRVn/P6NOyJTEAPGQOg6oH+TTidasBlu -i46g2j6vHqJc2dKHMeGZyes2sEMv0M33wIGFfYSqm19FXlNJftH/ceA4ETbdBPtP -EvaYhVNHwTrg2YlZX027m69BeeDp6U3OnX3OFPrnloPof4gEVUKr+mqPb2DAdPAg -jlJS+q1GjzGoldSdZbbztJLwEPBun89xK0a/SY+fjVvBVSIciOVHdTvkF2VmQRZ+ -yshdoH1DKFMit9O9wiHPH4T5GmH0sxGZABEBAAGJASUEGAEKAA8CGwwFAlKnORoF -CQFSqCgACgkQGYwa/VBRUt0ryggAhYsqnDY5BU/kadgYTAjnzxDK+HyXqZiPHpbC -LOwB/8v7YSmiCnQaeGNcOkd/Cp23nBhkCbYyHxRU5iok6qqVe3zaHCQ7PM9wwJeO -T0HK6TmGn05PXjMAlbseBWAeGA93Jl9dVRJWF4yUY5esw226mSsEtZgZjFSy51Gd -HRFsqL2UJjyVZQL46pizex0BK/XaKcaeAMPpbkXTEaJGwf2rFOSQLz9EmIqZVZZX -B9+SatOq2pfHM9Kc4SOl/Bz7m8rPuWOn13v8g2NgZN6dqxT00kKy2Vhnh+bZP9iX -/DUbuBYo1sbFae1gJO55A9KdWys1iXX5F5N7G5QCLCCIMgBvZLkBDQRSQylwAQgA -p7JEaTcViRP2IA0d6OrDFsMJm1CfH9kiWtIB3xzPn3McQts0Dff/ZqiUsUtRPUHX -rkW2x6WMRFLR6F9L0q5omv1QCw4kADKbOdUQaqumJa2Krz/0W9IWdGeP2II06LC3 -btT3k84oepsG8jTRdPoGPfOyIN9al0MrTRfvRtaEmPXYAQKudpbte9EF00cLpuAe -hoeP6QpzOscfu52NqKpYL5dsGB1QDid/Vva+MvZAYXFbOdV7yamHoY2qMimH9fj8 -wNBbL4SZGbhusLOXhBkgfdJQMyhshWV3nbzi5pxZqgK2qC3rimoxyBSQ3yjXxx22 -KZ/BESH1Vuup26pVxTtoNwARAQABiQElBBgBCgAPAhsgBQJSpzkpBQkBUV2zAAoJ -EBmMGv1QUVLdKGQH/Ar8l3+ViR0XMYCMt9GFnP/+7zYfUdqYBvgp0FxIRkxqY0Se -6ehVgjBZv91Vzha1zno+Ei6unp9ntP0FY9ftwLpqX/VJseG2DO834N9o7q02cgIf -pHJ2nwmnhndMeIy+vhrTzEzv/3cPb6EUOLApbULpJYoNLvbrrgmVrJN81ryKPybs -rcbUWosHxovXADuokvYTH2M8H47tSX0f+W//kDhhXWlQW/WUucYxMDY8Hw1xJWQa -rBMjA6cT3s4EZ2D0H09UeQEysKYWFprKud2Iy54Gq7242adDH/zNDV/weuBdPlnm -EuQYOGdfxTfFw/MRvkm8MVehOCu/MvEGqzeCC9E= -=T6TK +m/xFOH2iKSgx95LT2KGPpt6Hrf99MeYQPrBDufyinsrnPqzwACaMqMxMiI6tiQEc +BBABCgAGBQJSrz4ZAAoJECcHMw1AMMytWo4H/iC7DOJbIX1Fa3wrAIFxiLX46Mut +yR7eTs/6lUQZl9jVjzhbphTxUaSEW1g/K1hatbOEgBWsY3OgeAkZXXOgxN1Jdgc2 +nGsK1keoPh7h/pH7vhD0XZQZnF2fNKIYx9dyrA1xRFuQ2iXqTyfjfcYO8KVU6Mom +t1FS/YGebbBg3vnEVqtKNWZCtNOdhRGYJf/u421xoqg2sDJIe6EcLGEfw8Xy+q8H +Jq7OGOuHP4KCARMFyATeRQv7jWRdC1urCIIryVhY5VrJGM8809GAnGWT57v0pxsG +6yXvxvgTuXDyFhParv/Q/THOX8UdzRLFC6fyf3r/SukhD+VpTbVUd8Ah9L6JAT4E +EwECACgFAlJB3u8CGwMFCQCAxMEGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJ +EBmMGv1QUVLda7wIAKTnWdxkcjoeVMprDfp7+Z7QEaGaR8Wx8IUN3uVB+NZJBczS +1BwZ5rwnpi9xufPDNm+k5+5mA3KagJEmpc6G4JsQnrrVJ07qouh1G00v1BOq93/8 +3twQt5F3nsYjs5yujNcRJESNX5PnhwzyPCYTc5G4lBj2Jshq/DxxjqRnlEiT+LRU +Co8Lgdo2EHBLz3w/fhYvLhm9qlPzuKaqwmN+PE0IoKuan2C7WhMDRkldoASdhtBj +1blKXGdO5FlKDdQtRnK7jK+Ucz0j2tiS8miFc1HbG0tMvS57AQUjaAfh5ELSFq13 +gXjjkUdbzKnpG0YhoctDHonoPSCLrT4wWrmmZreIRgQTEQIABgUCUtj6GwAKCRCQ +aoVnZLJNU/iTAJsFFeY6LovkK++ekvdAA44G1qf96wCgnDghJNaI3CXKbtSMjhwC +HtMdBXGJARwEEAEIAAYFAlK0NDMACgkQvKAP1LIWjArh5Af/XZ9KfcmBNhOW9ePM +h6bNErWfxGyTDBPGcyO71F810yMt7GXfUn0jneYlcG8WJGBi7XJKDYHa3HFZDI+b +6KooYAtlcky6p+dmkDZiJNsBvrMpMQjP4QIFGBK7bOkLBHejFBiht8I6uSEmTPL9 +7oNyTkFo10ygSltsXWctih+pPsPaTGEjlInQVtuiVTVdr3xnu15wy9ay9c0cMgGH +asSTJeEq5ZaVVD3ODyw3LEhmOhIqWBYeLUIjNdvppDW3boKitWo0nqh9K0gOImdX +pAjCJCYbbQbKj2PgUhtFYbharMxdKHCuszQauLY/+tvtppyG9UuWVDwVkch0N8G5 +yonf8rkBDQRSQd7vAQgA3HTLlt1y1/kwa2Y9ioPY+0zoESWRUg0pfMg3c5N/52Sm +uNU4wZ1n/4FOpqqT5meYnUkOw5xVoO22TlfzRdAXuiEVZ/z+jTsiUxADxkDoOqB/ +k04nWrAZbouOoNo+rx6iXNnShzHhmcnrNrBDL9DN98CBhX2EqptfRV5TSX7R/3Hg +OBE23QT7TxL2mIVTR8E64NmJWV9Nu5uvQXng6elNzp19zhT655aD6H+IBFVCq/pq +j29gwHTwII5SUvqtRo8xqJXUnWW287SS8BDwbp/PcStGv0mPn41bwVUiHIjlR3U7 +5BdlZkEWfsrIXaB9QyhTIrfTvcIhzx+E+Rph9LMRmQARAQABiQElBBgBCgAPAhsM +BQJTguoABQkCLlkLAAoJEBmMGv1QUVLdwvAH/2a6O4YKBYomti04mqR0OUTnvfOd +/9QJqZVymv6ArHzVYiTF4VwHqjziW2P/rjzkpUWSIwG0z6B/AIcOz+E+Fv4u5r7b +S7k/iGwtfJ0ylEMpzBWOE9DeEw7zZ912U92zaBTJ1+2nz5IgcDnwpxV0mTfhf1Ah +/vQpzZUjD2lVEBfaGe+YzmC/IJG/WpUikxqxXgVclU3ecoEx5w87qfD02VT2l4Ck +o1dsmfwQcFaYnG5sayXpTqZl2ierAzLrBXPEpYoDTZqIJqAk9XlOW78ihLgv2O/J +b2lGiVf6Ev7cS///6zhfyKj0j3wa/xHffQuDa7r7TeUIsE3Qb8KbvtGr0Si5AQ0E +UkMpcAEIAKeyRGk3FYkT9iANHejqwxbDCZtQnx/ZIlrSAd8cz59zHELbNA33/2ao +lLFLUT1B165FtseljERS0ehfS9KuaJr9UAsOJAAymznVEGqrpiWtiq8/9FvSFnRn +j9iCNOiwt27U95POKHqbBvI00XT6Bj3zsiDfWpdDK00X70bWhJj12AECrnaW7XvR +BdNHC6bgHoaHj+kKczrHH7udjaiqWC+XbBgdUA4nf1b2vjL2QGFxWznVe8mph6GN +qjIph/X4/MDQWy+EmRm4brCzl4QZIH3SUDMobIVld5284uacWaoCtqgt64pqMcgU +kN8o18cdtimfwREh9VbrqduqVcU7aDcAEQEAAYkBJQQYAQoADwIbIAUCU4LqGQUJ +Ai0OpgAKCRAZjBr9UFFS3YUGB/9Jxu6OcKYfegT0LNaEZwhg/P1OetqX5SQfuhgt +Jzvq3bogLQBI4InkRmCHR2KU6Hj5hjU9fRYXJ6JJS/ReceX2zcva+MCcHvtL/Qnp +B7Womq8fjtbzhsbNRxhlCXX3/V8gDxmgPTdQ5mLW4LHuvE18GXHgzllkKJ+XXbBb +JnA7/Yk+RI6G4s8mru1kj5Lb4R0/Io+0INnct6ZUUMHzAR1pSJD/tPS+Gh5tsDG9 +ijD9nCO+1soHkFl5vUO+aFGHgQkmXWaFUWBk/uCuDd08NDu/Ai5zGo1gt0yfPFgN +9fuL1QPLE+OLY3p83BDtDZdEzmOLqL42Ntr38IczD+Gdvci/ +=yrrA -----END PGP PUBLIC KEY BLOCK----- diff --git a/global/overlay/etc/cosmos/keys/swold-BD115DD6.pub b/global/overlay/etc/cosmos/keys/swold-BD115DD6.pub deleted file mode 100644 index ecc5c8c..0000000 --- a/global/overlay/etc/cosmos/keys/swold-BD115DD6.pub +++ /dev/null @@ -1,65 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.12 (GNU/Linux) - -mQENBFJZKuEBCAC1UedRtQMx4ysvXcCbA/z2iGxu+DxcyB67nH3/r3iq6Zxo1HIF -JTjgMj0uetNoS2RQ6e0Xut9JWRQTP5OVeR89JbyjOuv8EPQ+OMo59qhcW/MnLMfG -3CfezFi75UeT5z4d8QG5No9allsqbsA7iuyy790rIDMc8dsYfnzUnDNO0p4SQ/18 -hyJl3ReZXIRiWJ/Sr7qNWX+vksu7aG+RiGKP6ORWVg7QZ/k5wTGSdYDKzrzkLndF -Yl93mdVEqC06kH+5+jDVBVBRV8x/YlFnuDP0QcX+nSxejU1sLKSH6K9chaeVsZaQ -Ra5n2HNIQHpQh7TNXWlfT+/xjpC3+YciDCDnABEBAAG0HFN0ZWZhbiBXb2xkIDxz -d29sZEBzdW5ldC5zZT6JAUEEEwECACsCGwMFCQHhM4AGCwkIBwMCBhUIAgkKCwQW -AgMBAh4BAheABQJSWVtIAhkBAAoJECbL9Wi9EV3WFMoH+QG1agR4m3jwtTO9/8as -MkYa8Zfqmh99JhbIDrEOdljyqJZLOzBgo53cQr57C159F0/EsdCKajM7bJSEdgDh -vlCIukD3NeuixN728Tgm9rj14kdYU/9ttZHOORK52tJZL6dZx60MamC3OpTdaMFD -1kvOLDg3jbBBwxxI9/iwUCudWfb+FTl3eTfClJC1gIBGWGNVCSrCqqjIvHvuiWFU -JQnT911DXZeowjWbJpsn7qVv/3TEx3ZVmE0ekZEO333Ky8/CEUNQcR018yUhcPKA -zJ7f/1YRuADdd35JCfntHzBzzMtTc2L3W54HEKYSRoRInuPdje6fqO2jxpGekD3q -qZC0LFN0ZWZhbiBXb2xkIChVbmRlck5FVCkgPHJhdGxlckB1bmRlcm5ldC5vcmc+ -iQE+BBMBAgAoBQJSWSuxAhsDBQkB4TOABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIX -gAAKCRAmy/VovRFd1rIWCACmBm9spo7dnR77HKMpKLy5DI9cdRC/zBkq3dxloLG0 -+k4HwSlUy0Nc77svcE0QhHeDlX8wenAVIXPSi1q0nVUN8J45qAYV3H52LlNzHRjO -8fjTVpsv/Cp7CmiuJR6I3Q6Nimmq4ZMSHPvqm6sJl0SohOk77Hg03jLt9/49Zolk -VTyK3anqZpCVPWs8Mbm8/ta33/9sA5P+JG++p6qSo2gJ7OE4lPFlrEpSJJ5KhsKD -HZnys4NfSPxU2uBIfAbo+qrS3rzksDuCn7agRPM2ZbIl9P6kchURfVKf1uzys9Kg -xFhspPBcszirZH+C65d/bNqUm1wRMM+9X+1IJeLyGbYitChTdGVmYW4gV29sZCAo -UHJpdmF0ZSkgPHJhdGxlckBzdGRlcnIuZXU+iQE+BBMBAgAoBQJSWStbAhsDBQkB -4TOABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAmy/VovRFd1oMcB/46oBGm -YDxVyjOgdrDVwZh3nKZLT7NfrKMckZdPCnG5RhbMzf0p2sK1whpoufgaGoOUpoNW -wLFXEmtoTWoogFHpcYjzkDpk3IcCz8WJObBlIqj9WgTg4m+gI03EQQ2IhN413zGM -j5STAhYtstZUgec0j5mPW3Gz/JMO+rCPkfwP1sXH0Is4g31hBoQM5xSfaFdChCIm -p/5VPjd35tkCXrJ+6/4nFWcIrm5hYnC3XEbw9YdDc4HRcYcowSygMSZ5PqiMATN+ -TJMfQMb/6e2GZz0S6wxIxiRMSBKS6e+vZNQgZbG9NtgdXjtlCnapED4hwBJsYmEO -y35MIpanYs58OOiCtDJTdGVmYW4gV29sZCAoTHVuYXItTGludXgpIDxyYXRsZXJA -bHVuYXItbGludXgub3JnPokBPgQTAQIAKAUCUlkrfwIbAwUJAeEzgAYLCQgHAwIG -FQgCCQoLBBYCAwECHgECF4AACgkQJsv1aL0RXdYRlgf/TGaMCfyiFrKlT2EgaSkv -xRrD/LsBcmRHIzkttKOVL0fIUpqyDHsSsxhnoYYVcuE8YdSzWyz8MkH3MYnaNKZw -jZSI6WdZxs/wRK9pT8Vesqow74xssQhSpGVR+ZsF7+ZrsMuBa+fNotio8TRnnHQM -7F9fskzp8q3KNi9CV9BWcwUS5RTs8imtklBK0ivLzmK4XozSjePuTZle4ubxdreU -FP0P2bBEKJKQ2lgPbV3fO4y8o+uyaVq87wFuz1Bkjw6FLOnPTeV0zZTwrexwP1Fc -4sdTGuS65A8waN0jtBCKiCGMp7Fhcc1PFYSeetNSDKD4P/gimRIFilpnRSelkMsk -oLkBDQRSWSrhAQgAwnlYChdrbwEGs06cpZSFFLa/CpB9k17YjQ2DPYfu0yrTAfrL -ImtWNhwgKHNUpWbBlM8m/NnUxyNyfAoSiTPfOZaBljP48bmfbLfCSZcdAXtLoo2z -mA2c/Ia1v48+zIi/r5T3U1r58yGmh5M9Oe2Q0mnzdvg65nyPab50rY94sW5xDrLz -ofuqbuvbpfYwbX8XfNl9JS2W3MY4NB1vbTaduWVyxdHtTr3O7PhFYBOSDZxz8Flf -AKmxFWNTlaPvwQ9G9lZBP/CZsow+Lm90ZJN56Skfzg8JZObUE/D2Qqu1MuR23gmG -PYrGcom4Q0jVbgI4qKIsk+EbW4ccYhY9ZxDRiwARAQABiQElBBgBAgAPBQJSWSrh -AhsMBQkB4TOAAAoJECbL9Wi9EV3WsP4H/j5Vo8/VjQKnjGJ7cbO8aQia66qVsJS1 -5JIOOKcdpRma30RLnrpYT/ni0JMQ3GlWCnPjDyvdwNfXIyrDnuLkyYV8QFcnJW9S -+QVWhooFAIoj6F3EeWyTXnQO8qWbGmyMXhERPXGE6ynF4dVbdAkQ9rEqXP2Dnqr3 -j2H6oG9Nsk7XZaaGr4FmMpHNhZONmBzhmuBXBn29IhPx+vQpcU6UYPmgM3va0qK3 -klLj96v6Oy1OPcsNX7Ud5atdho0bPV4kF/ddA1uiyjbTme9QHO8bQIlDZWWfSmg9 -AXLSIvrWF5JmX1o0HMX2UNW69GUn8ajdFGUk1bWQvxqmIEUUUqKo6NG5AQ0EUlks -swEIAMQf/spauLj012EKNdbxUR0PEWXch8gVPpCMnjtOSEJkI9l2AWXw0spQ4ETU -/iOvAH5/4b5JxD0Rg3Yyd/RWGNne3A+rZLV+iy7eOYDnqqFDYnpk0bVK9UOSBPr0 -dn7KjETNuSZa6GGVvtZZXMLeHUrlHGzxiv0YnMimCofk/MjvekwBLz0xg6OSGpLw -06LiicHeWUWfIVl5YMH6tePz9DqTW7YX0o3V8/F7cSKyRok86Q+fx0R7ek4CK74X -BbKKcfE71niDN/nldUTygV3fotGmgkavfa91JxI94KSysBGW4QeeQQeqDpxUtwOQ -qces8hO5cqPniL+Nh5gZ62KAS5UAEQEAAYkBJQQYAQIADwUCUlksswIbIAUJAeEz -gAAKCRAmy/VovRFd1lI2B/9V9gk3PSRNDfhvukRDhHdc6z8iuplSuUvXEc8NzZWJ -uUsSTFmS765ejWocXMhk0BgWiHweFvL5J8K64OBbV6Y8z5K24EZsO4jSj/GwRQGT -ZnjW4xetG/2eSCzt8DIy+dLQxNktXX/XWtKWmqWz3Wa03ZZ11QEIJ1SV5zoSCLYN -UDCH1RlQxp0dyKWqpdA2AO0qXEL76kOkNbntT/gDaHeutIGApyUV9kyqntxZvVSY -VAopF+o0M3mmudd6qJq/PCErEjFM8iD4C3EOtckC2o3PdBzIAubVonEfDX7gsYPn -HgMA8bkQmpez+RJ2cuY5ljdZs6iW+5pPWOU4dyqyNAmB -=iEp/ ------END PGP PUBLIC KEY BLOCK----- diff --git a/global/overlay/etc/cron.d/cosmos b/global/overlay/etc/cron.d/cosmos index 70af3a4..58b45af 100644 --- a/global/overlay/etc/cron.d/cosmos +++ b/global/overlay/etc/cron.d/cosmos @@ -1,4 +1,4 @@ SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -*/15 * * * * root test -f /etc/no-automatic-cosmos || (cosmos update ; cosmos apply) +*/15 * * * * root test -f /etc/no-automatic-cosmos || scriptherder --mode wrap --syslog --name cosmos -- /usr/local/bin/run-cosmos -v diff --git a/global/overlay/etc/cron.daily/scriptherder_cleanup b/global/overlay/etc/cron.daily/scriptherder_cleanup new file mode 100755 index 0000000..08ec7f5 --- /dev/null +++ b/global/overlay/etc/cron.daily/scriptherder_cleanup @@ -0,0 +1,10 @@ +#!/bin/sh +# +# Remove scriptherder data older than 7 days. +# + +DIR="/var/cache/scriptherder/" + +test -d ${DIR} || exit 0 + +find ${DIR} -type f -mtime +7 -print0 | xargs -0 rm -f diff --git a/global/overlay/etc/puppet/cosmos-modules.conf b/global/overlay/etc/puppet/cosmos-modules.conf index 7a95b31..bde1b61 100644 --- a/global/overlay/etc/puppet/cosmos-modules.conf +++ b/global/overlay/etc/puppet/cosmos-modules.conf @@ -22,3 +22,4 @@ mysql puppetlabs/mysql no apache puppetlabs/apache no pyff git://github.com/samlbits/puppet-pyff.git yes sunet-* postgresql git://github.com/SUNET/puppetlabs-postgresql.git yes sunet-* +dhcp git://github.com/SUNET/puppetlabs-dhcp.git yes sunet-* diff --git a/global/overlay/etc/puppet/facter/cosmos.rb b/global/overlay/etc/puppet/facter/cosmos.rb new file mode 100644 index 0000000..d810082 --- /dev/null +++ b/global/overlay/etc/puppet/facter/cosmos.rb @@ -0,0 +1,22 @@ +# +# Extract local Cosmos configuration +# +require 'facter' +Facter.add(:cosmos_repo) do + setcode do + Facter::Util::Resolution.exec("sh -c '. /etc/cosmos/cosmos.conf && echo $COSMOS_REPO'") + end +end + +Facter.add(:cosmos_tag_pattern) do + setcode do + Facter::Util::Resolution.exec("sh -c '. /etc/cosmos/cosmos.conf && echo $COSMOS_UPDATE_VERIFY_GIT_TAG_PATTERN'") + end +end + +Facter.add(:cosmos_repo_origin_url) do + setcode do + Facter::Util::Resolution.exec("sh -c '. /etc/cosmos/cosmos.conf && cd $COSMOS_REPO && git remote show -n origin | grep \"Fetch URL\" | awk \"{print \\$NF }\"'") + end +end + diff --git a/global/overlay/etc/puppet/hiera.yaml b/global/overlay/etc/puppet/hiera.yaml index cd619bb..782fa7f 100644 --- a/global/overlay/etc/puppet/hiera.yaml +++ b/global/overlay/etc/puppet/hiera.yaml @@ -1,13 +1,15 @@ --- -:backends: - yaml - - gpg +:backends: + - yaml + - gpg :logger: console -:hierarchy: - %{env}/%{location}/%{calling_module} - - %{env}/%{calling_module} - - secrets.yaml - - common +:hierarchy: + - "%{env}/%{location}/%{calling_module}" + - "%{env}/%{calling_module}" + - secrets.yaml + - common :yaml: diff --git a/global/overlay/etc/puppet/manifests/cosmos-site.pp b/global/overlay/etc/puppet/manifests/cosmos-site.pp index 34ff4cb..8835226 100644 --- a/global/overlay/etc/puppet/manifests/cosmos-site.pp +++ b/global/overlay/etc/puppet/manifests/cosmos-site.pp @@ -44,11 +44,11 @@ node default { node 'sto-tug-kvm1.swamid.se' { - + package {'python-vm-builder': ensure => 'installed', - } -> - + } -> + cosmos::dhcp_kvm { 'registry.swamid.se': mac => '52:54:00:52:53:0b', repo => 'git://git.nordu.net/sunet-ops.git', @@ -286,36 +286,100 @@ class sunet-dhcp-hosts { # eduID Development subnets - dhcp::pool {'eduid-tug-dev': - network => '194.68.13.128', - mask => '255.255.255.224', - gateway => '194.68.13.129', - range => '', - options => 'domain-name-servers 109.105.111.31, 109.105.110.31', - } + #dhcp::pool {'eduid-tug-dev': + # network => '194.68.13.128', + # mask => '255.255.255.224', + # gateway => '194.68.13.129', + # range => '', + # options => 'domain-name-servers 109.105.111.31, 109.105.110.31', + #} - dhcp::pool {'eduid-fre-dev': - network => '194.68.13.160', - mask => '255.255.255.224', - gateway => '194.68.13.161', - range => '', - options => 'domain-name-servers 109.105.111.31, 109.105.110.31', + #dhcp::pool {'eduid-fre-dev': + # network => '194.68.13.160', + # mask => '255.255.255.224', + # gateway => '194.68.13.161', + # range => '', + # options => 'domain-name-servers 109.105.111.31, 109.105.110.31', + #} + + dhcp::pool {'eduid-dev-tug': + network => '130.242.130.192', + mask => '255.255.255.224', + gateway => '130.242.130.193', + range => '' } + # One big subnet used for now + #dhcp::pool {'eduid-dev-tug-IdP': + # network => '130.242.130.192', + # mask => '255.255.255.248', + # gateway => '130.242.130.201', + # range => '' + #} + + # One big subnet used for now + #dhcp::pool {'eduid-dev-tug-auth': + # network => '130.242.130.200', + # mask => '255.255.255.248', + # gateway => '130.242.130.201', + # range => '' + #} + + # One big subnet used for now + #dhcp::pool {'eduid-dev-tug-other': + # network => '130.242.130.208', + # mask => '255.255.255.240', + # gateway => '130.242.130.209', + # range => '' + #} + # eduID TUG development hosts - dhcp::host { 'idp-tug-1': mac => "52:54:00:a0:00:92", ip => "194.68.13.146" } + dhcp::host { 'worker-fre-1': mac => "52:54:00:a0:01:c4", ip => "130.242.130.196" } + + dhcp::host { 'auth-fre-1_eth0': mac => "78:45:c4:f7:91:67", ip => "130.242.130.204", hostname => 'auth-fre-1'; } + dhcp::host { 'auth-fre-1_eth1': mac => "78:45:c4:f7:91:68", ip => "130.242.130.204", hostname => 'auth-fre-1'; } + + dhcp::host { 'auth-tug-1_eth0': mac => "78:45:c4:f8:43:c5", ip => "130.242.130.205", hostname => 'auth-tug-1'; } + dhcp::host { 'auth-tug-1_eth1': mac => "78:45:c4:f8:43:c6", ip => "130.242.130.205", hostname => 'auth-tug-1'; } + + dhcp::host { 'admin-tug-1': mac => "52:54:00:a0:01:d4", ip => "130.242.130.212" } + + dhcp::host { 'dash-fre-1': mac => "52:54:00:a0:01:d5", ip => "130.242.130.213" } + + dhcp::host { 'idp-fre-1': mac => "52:54:00:a0:01:d6", ip => "130.242.130.214" } + + dhcp::host { 'idp-tug-1': mac => "52:54:00:a0:01:d7", ip => "130.242.130.215" } + + dhcp::host { 'kvm-fre-1_eth0': mac => "78:45:c4:f8:45:15", ip => "130.242.130.216", hostname => 'kvm-fre-1'; } + dhcp::host { 'kvm-fre-1_eth1': mac => "78:45:c4:f8:45:16", ip => "130.242.130.216", hostname => 'kvm-fre-1'; } + + dhcp::host { 'kvm-tug-1_eth0': mac => "78:45:c4:f8:47:be", ip => "130.242.130.217", hostname => 'kvm-tug-1'; } + dhcp::host { 'kvm-tug-1_eth1': mac => "78:45:c4:f8:47:bf", ip => "130.242.130.217", hostname => 'kvm-tug-1'; } + + dhcp::host { 'monitor-fre-1': mac => "52:54:00:a0:01:da", ip => "130.242.130.218" } - dhcp::host { 'testvm-tug-1': mac => "52:54:00:11:22:33", ip => "194.68.13.136" } + dhcp::host { 'mq-fre-1': mac => "52:54:00:a0:01:db", ip => "130.242.130.219" } - dhcp::host { 'userdb-tug-1': mac => "52:54:00:93:22:29", ip => "194.68.13.132" } - dhcp::host { 'userdb-tug-2': mac => "52:54:00:17:13:ff", ip => "194.68.13.133" } + dhcp::host { 'userdb-fre-1': mac => "52:54:00:a0:01:dc", ip => "130.242.130.220" } + + dhcp::host { 'userdb-tug-1': mac => "52:54:00:a0:01:dd", ip => "130.242.130.221" } + + dhcp::host { 'userdb-tug-2': mac => "52:54:00:a0:01:de", ip => "130.242.130.222" } + + + #dhcp::host { 'idp-tug-1': mac => "52:54:00:a0:00:92", ip => "194.68.13.146" } + + #dhcp::host { 'testvm-tug-1': mac => "52:54:00:11:22:33", ip => "194.68.13.136" } + + #dhcp::host { 'userdb-tug-1': mac => "52:54:00:93:22:29", ip => "194.68.13.132" } + #dhcp::host { 'userdb-tug-2': mac => "52:54:00:17:13:ff", ip => "194.68.13.133" } # eduID FRE development hosts - dhcp::host { 'idp-fre-1': mac => "52:54:00:a1:00:b2", ip => "194.68.13.178" } + #dhcp::host { 'idp-fre-1': mac => "52:54:00:a1:00:b2", ip => "194.68.13.178" } - dhcp::host { 'dash-fre-1': mac => "52:54:00:a2:00:a7", ip => "194.68.13.167" } + #dhcp::host { 'dash-fre-1': mac => "52:54:00:a2:00:a7", ip => "194.68.13.167" } - dhcp::host { 'userdb-fre-1': mac => "52:54:00:17:13:f6", ip => "194.68.13.164" } + #dhcp::host { 'userdb-fre-1': mac => "52:54:00:17:13:f6", ip => "194.68.13.164" } # SUNET TUG hosts @@ -325,6 +389,10 @@ class sunet-dhcp-hosts { dhcp::host { 'md-master.reep': mac => "52:54:00:39:8d:ac", ip => "130.242.125.83" } dhcp::host { 'lobo2.lab': mac => "52:54:00:5e:72:91", ip => "130.242.125.86" } + # SUNET TUG eduID hosts (KVM host cdr1.sunet.se) + dhcp::host { 'backup-tug-3': mac => "52:54:00:f2:7d:54", ip => "130.242.125.84" } + dhcp::host { 'proxy-tug-3': mac => "52:54:00:f2:7d:55", ip => "130.242.125.85" } + # SWAMID production dhcp::host { 'registry.swamid': mac => "52:54:00:52:53:0b", ip => "130.242.125.90" } } @@ -481,13 +549,47 @@ node 'wp.sunet.se' { } } +node 'cdr1.sunet.se' { + + package {'python-vm-builder': + ensure => 'installed', + } -> + + cosmos::dhcp_kvm { 'backup-tug-3.eduid.se': + mac => '52:54:00:f2:7d:54', + repo => 'git://git.nordu.net/eduid-ops.git', + tagpattern => 'eduid-v3', + cpus => '1', + memory => '512', + suite => 'trusty', + extras => '--addpkg linux-image-generic --tmpfs -', + } + + cosmos::dhcp_kvm { 'proxy-tug-3.eduid.se': + mac => '52:54:00:f2:7d:55', + repo => 'git://git.nordu.net/eduid-ops.git', + tagpattern => 'eduid-v3', + cpus => '1', + memory => '512', + suite => 'trusty', + extras => '--addpkg linux-image-generic --tmpfs -', + } + +} + class sunet-cdr { + # Listen on br0 if it exists (cdr1), otherwise bond0 (cdr2). + $interface = $::ipaddress_br0 ? { + undef => 'bond0', + default => 'br0', + } + class { 'dhcp': dnsdomain => [ 'eduid.se','sunet.se','swamid.se' ], nameservers => ['130.242.80.14','130.242.80.99'], ntpservers => ['ntp1.nordu.net','ntp2.nordu.net','Time1.Stupi.SE'], - interfaces => ['bond0'], + interfaces => [$interface], #pxeserver => '130.242.125.5', #pxefilename => 'pxelinux.0' } diff --git a/global/overlay/etc/puppet/modules/sunet/manifests/server.pp b/global/overlay/etc/puppet/modules/sunet/manifests/server.pp index 875dc69..14df323 100644 --- a/global/overlay/etc/puppet/modules/sunet/manifests/server.pp +++ b/global/overlay/etc/puppet/modules/sunet/manifests/server.pp @@ -37,4 +37,51 @@ define sunet::server() { notify => Service['ssh'], } + # already declared in puppet-cosmos/manifests/ntp.pp + #service { 'ntp': + # ensure => 'running', + #} + + # Don't use pool.ntp.org servers, but rather DHCP provided NTP servers + line { 'no_pool_ntp_org_servers': + file => '/etc/ntp.conf', + line => '^server .*\.pool\.ntp\.org', + ensure => 'comment', + notify => Service['ntp'], + } + + file { '/var/cache/scriptherder': + ensure => 'directory', + path => '/var/cache/scriptherder', + mode => '1777', # like /tmp, so user-cronjobs can also use scriptherder + } + + +} + +# from http://projects.puppetlabs.com/projects/puppet/wiki/Simple_Text_Patterns/5 +define line($file, $line, $ensure = 'present') { + case $ensure { + default : { err ( "unknown ensure value ${ensure}" ) } + present: { + exec { "/bin/echo '${line}' >> '${file}'": + unless => "/bin/grep -qFx '${line}' '${file}'" + } + } + absent: { + exec { "/usr/bin/perl -ni -e 'print unless /^\\Q${line}\\E\$/' '${file}'": + onlyif => "/bin/grep -qFx '${line}' '${file}'" + } + } + uncomment: { + exec { "/bin/sed -i -e'/${line}/s/^#\\+//' '${file}'": + onlyif => "/bin/grep '${line}' '${file}' | /bin/grep '^#' | /usr/bin/wc -l" + } + } + comment: { + exec { "/bin/sed -i -e'/${line}/s/^\\(.\\+\\)$/#\\1/' '${file}'": + onlyif => "/usr/bin/test `/bin/grep '${line}' '${file}' | /bin/grep -v '^#' | /usr/bin/wc -l` -ne 0" + } + } + } } diff --git a/global/overlay/etc/puppet/puppet.conf b/global/overlay/etc/puppet/puppet.conf index a269892..96f7b44 100644 --- a/global/overlay/etc/puppet/puppet.conf +++ b/global/overlay/etc/puppet/puppet.conf @@ -3,7 +3,9 @@ logdir=/var/log/puppet vardir=/var/lib/puppet ssldir=/var/lib/puppet/ssl rundir=/var/run/puppet -factpath=$vardir/lib/facter +# factpath is supposed to be colon-delimeted, but that does not appear to work +# (tested with 'strace -f facter --puppet something' - does not split on colon in Puppet 3.4.2). +factpath=/etc/puppet/facter templatedir=$confdir/templates node_terminus = exec external_nodes = /etc/puppet/cosmos_enc.py diff --git a/global/overlay/etc/scriptherder/check/cosmos.ini b/global/overlay/etc/scriptherder/check/cosmos.ini new file mode 100644 index 0000000..b44a537 --- /dev/null +++ b/global/overlay/etc/scriptherder/check/cosmos.ini @@ -0,0 +1,3 @@ +[check] +ok = exit_status=0, max_age=35m +warning = exit_status=0, max_age=1h diff --git a/global/overlay/usr/local/bin/scriptherder b/global/overlay/usr/local/bin/scriptherder index c11383a..1e00ec0 100755 --- a/global/overlay/usr/local/bin/scriptherder +++ b/global/overlay/usr/local/bin/scriptherder @@ -68,6 +68,28 @@ exit_status = {'OK': 0, } +class ScriptHerderError(Exception): + """ + Base exception class for scriptherder. + """ + + def __init__(self, reason, filename): + self.reason = reason + self.filename = filename + + +class JobLoadError(ScriptHerderError): + """ + Raised when loading a job file fails. + """ + + +class CheckLoadError(ScriptHerderError): + """ + Raised when loading a check file fails. + """ + + class Job(object): """ Representation of an execution of a job. @@ -109,6 +131,21 @@ class Job(object): exit = self.exit_status, ) + def status_summary(self): + """ + Return short string with status of job. + + E.g. 'name[exit=0,age=19h]' + """ + if self._end_time is None or self._start_time is None: + return '{name}[not_running]'.format(name = self.name) + age = _time_to_str(time.time() - self._start_time) + return '{name}[exit={exit_status},age={age}]'.format( + name = self.name, + exit_status = self._exit_status, + age = age, + ) + @property def name(self): """ @@ -167,11 +204,10 @@ class Job(object): @rtype: string """ + if self._end_time is None or self._start_time is None: + return 'NaN' duration = self._end_time - self._start_time - if duration < 1: - # milliseconds - return '{:0.3f}ms'.format(duration * 1000) - return '{:0.3f}s'.format(duration) + return _time_to_str(duration) @property def exit_status(self): @@ -326,7 +362,7 @@ class Job(object): #self._output_size = data.get('output_size') # currently not used in scriptherder self._filename = filename else: - raise AssertionError('Unknown version in file {!r}: {!r}'.format(filename, data.get('version'))) + raise JobLoadError('Unknown version: {!r}'.format(data.get('version')), filename=filename) return self @@ -355,7 +391,7 @@ class Check(object): self.logger = logger self.config = ConfigParser.ConfigParser(_check_defaults) if not self.config.read([filename]): - raise ValueError("Failed loading config file {!r}".format(filename)) + raise ScriptHerderError('Failed loading config file', filename) _section = 'check' self._ok_criteria = [x.strip() for x in self.config.get(_section, 'ok').split(',')] self._warning_criteria = [x.strip() for x in self.config.get(_section, 'warning').split(',')] @@ -422,6 +458,130 @@ class Check(object): return False +class CheckStatus(object): + """ + Aggregated status of job invocations for --mode check. + + Attributes: + + checks_ok: List of checks in OK state ([Job()]). + checks_warning: List of checks in WARNING state ([Job()]). + checks_critical: List of checks in CRITICAL state ([Job()]). + """ + + def __init__(self, args, logger): + """ + @param args: Parsed command line arguments + @param logger: logging logger + """ + + self.checks_ok = [] + self.checks_warning = [] + self.checks_critical = [] + + self._jobs = _get_job_results(args, logger) + # group the jobs by their name + _by_name = {} + for this in self._jobs: + if this.name not in _by_name: + _by_name[this.name] = [] + _by_name[this.name].append(this) + self._jobs_by_name = _by_name + + self._job_count = len(_by_name) + + self._check_running_jobs(args, logger) + if not args.cmd: + self._check_not_running(args, logger) + + def _check_running_jobs(self, args, logger): + """ + Look for job execution entrys (parsed into Job() instances), group them + per check name and determine the status. For each group, append status + to one of the three aggregate status lists of this object (checks_ok, + checks_warning or checks_critical). + + @param args: Parsed command line arguments + @param logger: logging logger + """ + # determine total check status based on all logged invocations of this job + for (name, jobs) in self._jobs_by_name.items(): + # Load the evaluation criterias for this job + check_filename = os.path.join(args.checkdir, name + '.ini') + logger.debug("Loading check definition from {!r}".format(check_filename)) + try: + check = Check(check_filename, logger) + except ScriptHerderError as exc: + logger.warning("Failed loading check: {!r}".format(exc), exc_info=True) + raise CheckLoadError('Failed loading check', filename = check_filename) + + # Sort jobs, oldest first + jobs = sorted(jobs, key=lambda x: x.start_time) + logger.debug("Checking {!r}: {!r}".format(name, jobs)) + + jobs_ok = [] + jobs_warning = [] + jobs_critical = [] + for job in jobs: + if check.job_is_ok(job): + jobs_ok.append(job) + elif check.job_is_warning(job): + jobs_warning.append(job) + else: + jobs_critical.append(job) + + logger.debug("Raw status OK : {!r}".format(jobs_ok)) + logger.debug("Raw status WARN : {!r}".format(jobs_warning)) + logger.debug("Raw status CRITICAL: {!r}".format(jobs_critical)) + + # add most recent job status to the totals + if jobs_ok: + self.checks_ok.append(jobs_ok[-1]) + elif jobs_warning: + self.checks_warning.append(jobs_warning[-1]) + else: + self.checks_critical.append(jobs_critical[-1]) + + def _check_not_running(self, args, logger): + """ + Look for job execution entrys (parsed into Job() instances), group them + per check name and determine the status. For each group, append status + to one of the three aggregate status lists of this object (checks_ok, + checks_warning or checks_critical). + + @param args: Parsed command line arguments + @param logger: logging logger + """ + files = [f for f in os.listdir(args.checkdir) if os.path.isfile(os.path.join(args.checkdir, f))] + for this in files: + if not this.endswith('.ini'): + continue + filename = os.path.join(args.checkdir, this) + logger.debug("Loading check definition from {!r}".format(filename)) + try: + # validate check loads + Check(filename, logger) + except ValueError as exc: + logger.warning("Failed loading check: {!r}".format(exc), exc_info=True) + raise CheckLoadError(filename = filename) + name = this[:-4] # remove the '.ini' suffix + if name not in self._jobs_by_name: + logger.debug('Check {!r} (filename {!r}) not found in jobs'.format(name, filename)) + job = Job(name=name) + self.checks_critical.append(job) + self._job_count += 1 + else: + logger.debug('Check {!r} has {!r} logged results'.format(name, len(self._jobs_by_name[name]))) + + def num_jobs(self): + """ + Return number of jobs processed. This is number of different jobs running + not running. + + @rtype: int + """ + return self._job_count + + def job_from_file(filename): """ Recreate Job() instance from saved file. @@ -488,6 +648,7 @@ def parse_args(defaults): ) args = parser.parse_args() + return args @@ -537,88 +698,61 @@ def mode_check(args, logger): @param args: Parsed command line arguments @param logger: logging logger """ - jobs = _get_job_results(args, logger) - # group the jobs by their name - by_name = {} - for this in jobs: - if this.name not in by_name: - by_name[this.name] = [] - by_name[this.name].append(this) - - total_ok = [] - total_warning = [] - total_critical = [] - - # determine total check status based on all logged invocations of this job - for (name, jobs) in by_name.items(): - # Sort jobs, oldest first - jobs = sorted(jobs, key=lambda x: x.start_time) - # Load the evaluation criterias for this job - check_filename = os.path.join(args.checkdir, name + '.ini') - logger.debug("Loading check definition from {!r}".format(check_filename)) - check = Check(check_filename, logger) - logger.debug("Checking {!r}: {!r}".format(name, jobs)) - - jobs_ok = [] - jobs_warning = [] - jobs_critical = [] - for job in jobs: - if check.job_is_ok(job): - jobs_ok.append(job) - elif check.job_is_warning(job): - jobs_warning.append(job) - else: - jobs_critical.append(job) - logger.debug("Raw status OK : {!r}".format(jobs_ok)) - logger.debug("Raw status WARN : {!r}".format(jobs_warning)) - logger.debug("Raw status CRITICAL: {!r}".format(jobs_critical)) - if jobs_ok: - total_ok.append(jobs_ok[-1]) - elif jobs_warning: - total_warning.append(jobs_warning[-1]) - else: - total_critical.append(jobs_critical[-1]) + try: + status = CheckStatus(args, logger) + except CheckLoadError as exc: + print("UNKNOWN: Failed loading check from file '{!s}' ({!s})".format(exc.filename, exc.reason)) + return exit_status['UNKNOWN'] if args.cmd: # Single job check requested, output detailed information - if total_ok: - print('OK: {!s}'.format(total_ok[-1])) + if status.checks_ok: + print('OK: {!s}'.format(status.checks_ok[-1])) return exit_status['OK'] - if total_warning: - print('WARNING: {!s}'.format(total_warning[-1])) + if status.checks_warning: + print('WARNING: {!s}'.format(status.checks_warning[-1])) return exit_status['WARNING'] - if total_critical: - print('CRITICAL: {!s}'.format(total_critical[-1])) + if status.checks_critical: + print('CRITICAL: {!s}'.format(status.checks_critical[-1])) return exit_status['CRITICAL'] print "UNKNOWN - no jobs found for {!r}?".format(args.cmd) return exit_status['UNKNOWN'] - # When not looking at multiple jobs at once, logic gets a bit reversed - if ANY + # When looking at multiple jobs at once, logic gets a bit reversed - if ANY # job invocation is CRITICAL/WARNING, the aggregate message given to # Nagios will have to be a failure. - if total_critical: - print("CRITICAL: {num} job(s) in this state: {names}".format( - num = len(total_critical), - names = ', '.join([str(x.name) for x in total_critical]), - )) + if status.checks_critical: + print('CRITICAL: {!s}'.format( + _status_summary(status.num_jobs(), status.checks_critical))) return exit_status['CRITICAL'] - if total_warning: - print("WARNING: {num} job(s) in this state: {names}".format( - num = len(total_warning), - names = ', '.join([str(x.name) for x in total_warning]), - )) + if status.checks_warning: + print('WARNING: {!s}'.format( + _status_summary(status.num_jobs(), status.checks_warning))) return exit_status['WARNING'] - if total_ok: - print("OK: {num} job(s) in this state: {names}".format( - num = len(total_ok), - names = ', '.join([x.name for x in total_ok]), - )) + if status.checks_ok: + print('OK: {!s}'.format( + _status_summary(status.num_jobs(), status.checks_ok))) return exit_status['OK'] print "UNKNOWN - no jobs found?" return exit_status['UNKNOWN'] +def _status_summary(num_jobs, failed): + """ + String format routine used in output of checks status. + """ + fmt = '1 job in this state: {summary}' + if len(failed) == 1: + fmt = '{jobs}/{num_jobs} job in this state: {summary}' + + summary = ', '.join(sorted([str(x.status_summary()) for x in failed])) + return fmt.format(jobs = len(failed), + num_jobs = num_jobs, + summary = summary, + ) + + def _get_job_results(args, logger): """ Load all jobs matching any specified name on the command line. @@ -634,7 +768,10 @@ def _get_job_results(args, logger): if not this.endswith('.json'): continue filename = os.path.join(args.datadir, this) - job = job_from_file(filename) + try: + job = job_from_file(filename) + except JobLoadError as exc: + logger.warning("Failed loading job file '{!s}' ({!s})".format(exc.filename, exc.reason)) if args.cmd: if args.cmd[0] != job.name: logger.debug("Skipping '{!s}' not matching '{!s}' (file {!s})".format(job.name, args.cmd[0], filename)) @@ -666,6 +803,27 @@ def _parse_time_value(value): return num +def _time_to_str(value): + """ + Format number of seconds to short readable string. + + @type value: float or int + + @rtype: string + """ + if value < 1: + # milliseconds + return '{:0.3f}ms'.format(value * 1000) + if value < 60: + return '{!s}s'.format(int(value)) + if value < 3600: + return '{!s}m'.format(int(value)) + if value < 86400: + return '{!s}h'.format(int(value / 3600)) + days = int(value / 86400) + return '{!s}d{!s}h'.format(days, int((value % 86400) / 3600)) + + def main(myname = 'scriptherder', args = None, logger = None, defaults=_defaults): """ Main entry point for either wrapping a script, or checking the status of it. @@ -699,6 +857,10 @@ def main(myname = 'scriptherder', args = None, logger = None, defaults=_defaults syslog_h.setFormatter(formatter) logger.addHandler(syslog_h) + if args.name and args.mode != 'wrap': + logger.error('Argument --name only applicable for --mode wrap') + return False + if args.mode == 'wrap': return mode_wrap(args, logger) elif args.mode == 'ls': diff --git a/global/post-tasks.d/015cosmos-trust b/global/post-tasks.d/015cosmos-trust index 447d875..5c3359b 100755 --- a/global/post-tasks.d/015cosmos-trust +++ b/global/post-tasks.d/015cosmos-trust @@ -4,12 +4,15 @@ if [ -z "$COSMOS_KEYS" ]; then COSMOS_KEYS=/etc/cosmos/keys fi +# Install new keys discovered in the $COSMOS_KEYS directory for k in $COSMOS_KEYS/*.pub; do fp=`cosmos gpg --with-colons --with-fingerprint < $k| awk -F: '$1 == "pub" {print $5}'` - cosmos gpg --with-colons --fingerprint | grep -q ":$fp:" || cosmos gpg --import < $k + # The removal of any ^pub:e: entrys means to ignore expired keys - thereby importing them again. + cosmos gpg --with-colons --fingerprint | grep -v "^pub:e:" | grep -q ":$fp:" || cosmos gpg --import < $k done -for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5}'`; do +# Delete keys no longer present in $COSMOS_KEYS directory +for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5 }'`; do seen="no" for k in $COSMOS_KEYS/*.pub; do cosmos gpg --with-colons --with-fingerprint < $k | grep -q ":$fp:" && seen="yes" |