From 107977330f9e81e198eb2b663d461ce84e8f55e1 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 13:20:02 +0100 Subject: [PATCH 01/36] Consistency Initial commit of `shfmt` and `shellcheck`. More to come. --- addhost | 68 +-- edit-secrets | 473 +++++++++--------- global/post-tasks.d/010cosmos-modules | 20 +- global/post-tasks.d/010fix-ssh-perms | 24 +- global/post-tasks.d/014set-cosmos-permissions | 6 +- global/post-tasks.d/015cosmos-trust | 78 +-- global/post-tasks.d/018packages | 182 +++---- global/post-tasks.d/030puppet | 24 +- global/post-tasks.d/099autoremove | 6 +- global/post-tasks.d/999reboot | 64 +-- global/pre-tasks.d/014set-cosmos-permissions | 6 +- global/pre-tasks.d/015set-overlay-permissions | 14 +- global/pre-tasks.d/020common-tools | 10 +- global/pre-tasks.d/030puppet | 34 +- global/pre-tasks.d/040hiera-eyaml | 24 +- global/pre-tasks.d/040hiera-gpg | 40 +- 16 files changed, 543 insertions(+), 530 deletions(-) diff --git a/addhost b/addhost index ef68eeb..3645b72 100755 --- a/addhost +++ b/addhost @@ -5,40 +5,48 @@ cmd_do_bootstrap="no" cmd_fqdn="" function usage() { - echo "Usage: $0 [-h] [-b] [-n fqdn] [--] []" - echo " -h show help" - echo " -b bootstrap (using ssh)" - echo " -n specify FQDN (if not the same as )" - echo "" - echo " can be an IP number, or something that resolves to one" + echo "Usage: $0 [-h] [-b] [-n fqdn] [--] []" + echo " -h show help" + echo " -b bootstrap (using ssh)" + echo " -n specify FQDN (if not the same as )" + echo "" + echo " can be an IP number, or something that resolves to one" } while getopts "bhn:p:" this; do - case "${this}" in - h) usage; exit 0;; - b) cmd_do_bootstrap="yes" ;; - n) cmd_fqdn="${OPTARG}" ;; - p) cmd_proxy="${OPTARG}" ;; - *) echo "Unknown option ${this}"; echo ""; usage; exit 1;; - esac + case "${this}" in + h) + usage + exit 0 + ;; + b) cmd_do_bootstrap="yes" ;; + n) cmd_fqdn="${OPTARG}" ;; + p) cmd_proxy="${OPTARG}" ;; + *) + echo "Unknown option ${this}" + echo "" + usage + exit 1 + ;; + esac done -shift $((OPTIND-1)) +shift $((OPTIND - 1)) if [[ ! $cmd_hostname ]]; then - cmd_hostname="$1" + cmd_hostname="$1" fi if [[ ! $cmd_fqdn ]]; then - cmd_fqdn="$cmd_hostname" + cmd_fqdn="$cmd_hostname" fi if test -z "$cmd_hostname"; then - usage - exit 1 + usage + exit 1 fi if [[ -n $cmd_proxy ]]; then - proxyjump="-o ProxyJump=${cmd_proxy}" + proxyjump="-o ProxyJump=${cmd_proxy}" fi test -f cosmos.conf && . ./cosmos.conf @@ -49,21 +57,21 @@ rrepo=${repo:="$defrepo"} rtag=${tag:="changeme"} if [[ ! $rrepo ]]; then - echo "$0: repo not set in cosmos.conf and no git remote named '${_remote}' found" - exit 1 + echo "$0: repo not set in cosmos.conf and no git remote named '${_remote}' found" + exit 1 fi if [ ! -d "$cmd_fqdn" ]; then - cp -pr default "$cmd_fqdn" - git add "$cmd_fqdn" - git commit -m "$cmd_fqdn added" "$cmd_fqdn" - ./bump-tag + cp -pr default "$cmd_fqdn" + git add "$cmd_fqdn" + git commit -m "$cmd_fqdn added" "$cmd_fqdn" + ./bump-tag fi if [ "$cmd_do_bootstrap" = "yes" ]; then - cosmos_deb=$(find apt/ -maxdepth 1 -name 'cosmos_*.deb' | sort -V | tail -1) - scp $proxyjump "$cosmos_deb" apt/bootstrap-cosmos.sh root@"$cmd_hostname": - ssh root@"$cmd_hostname" $proxyjump ./bootstrap-cosmos.sh "$cmd_fqdn" "$rrepo" "$rtag" - ssh root@"$cmd_hostname" $proxyjump cosmos update - ssh root@"$cmd_hostname" $proxyjump cosmos apply + cosmos_deb=$(find apt/ -maxdepth 1 -name 'cosmos_*.deb' | sort -V | tail -1) + scp $proxyjump "$cosmos_deb" apt/bootstrap-cosmos.sh root@"$cmd_hostname": + ssh root@"$cmd_hostname" $proxyjump ./bootstrap-cosmos.sh "$cmd_fqdn" "$rrepo" "$rtag" + ssh root@"$cmd_hostname" $proxyjump cosmos update + ssh root@"$cmd_hostname" $proxyjump cosmos apply fi diff --git a/edit-secrets b/edit-secrets index da40946..1938726 100755 --- a/edit-secrets +++ b/edit-secrets @@ -24,39 +24,38 @@ TMPFILE=$(mktemp edit-secrets.XXXXXXXXXX) TMPFILE2=$(mktemp edit-secrets.XXXXXXXXXX) if [ ! -f "$TMPFILE" ]; then - echo "$TMPFILE" - echo "$0: Failed creating temporary file" - exit 1 + echo "$TMPFILE" + echo "$0: Failed creating temporary file" + exit 1 fi if [ ! -f "$TMPFILE2" ]; then - echo "$TMPFILE2" - echo "$0: Failed creating temporary file 2" - exit 1 + echo "$TMPFILE2" + echo "$0: Failed creating temporary file 2" + exit 1 fi trap 'rm -f "$TMPFILE" "$TMPFILE2"' EXIT - if [[ ! $1 ]]; then - # deliberately don't mention the --on-host argument - echo "Syntax: $0 fqdn" - exit 1 + # deliberately don't mention the --on-host argument + echo "Syntax: $0 fqdn" + exit 1 fi function patch_broken_eyaml { - # - # Ubuntu 22.04 (jammy) has a broken hiera-eyaml package, a bug report - # exists here: https://bugs.launchpad.net/ubuntu/+source/hiera-eyaml/+bug/1974059 - # - - if [ "$(lsb_release -cs)" == "jammy" ]; then - plugins_file="/usr/share/rubygems-integration/all/gems/hiera-eyaml-3.2.2/lib/hiera/backend/eyaml/plugins.rb" - if [ -f $plugins_file ]; then - # We only want to try patching the file if it is the known broken version - bad_sum="1d0f14765ebcfcdae300d8ac5d715845ef9b283345d19114a23d96161556618f" - sum=$(sha256sum $plugins_file | awk '{print $1}') - if [ "$sum" == "$bad_sum" ]; then - patch --fuzz=0 --directory=/ --strip=0 <<'EOF' + # + # Ubuntu 22.04 (jammy) has a broken hiera-eyaml package, a bug report + # exists here: https://bugs.launchpad.net/ubuntu/+source/hiera-eyaml/+bug/1974059 + # + + if [ "$(lsb_release -cs)" == "jammy" ]; then + plugins_file="/usr/share/rubygems-integration/all/gems/hiera-eyaml-3.2.2/lib/hiera/backend/eyaml/plugins.rb" + if [ -f $plugins_file ]; then + # We only want to try patching the file if it is the known broken version + bad_sum="1d0f14765ebcfcdae300d8ac5d715845ef9b283345d19114a23d96161556618f" + sum=$(sha256sum $plugins_file | awk '{print $1}') + if [ "$sum" == "$bad_sum" ]; then + patch --fuzz=0 --directory=/ --strip=0 <<'EOF' --- /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.2.2/lib/hiera/backend/eyaml/plugins.rb.orig 2023-01-18 08:20:22.140338419 +0000 +++ /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.2.2/lib/hiera/backend/eyaml/plugins.rb 2023-01-18 08:21:05.654053501 +0000 @@ -32,6 +32,7 @@ @@ -68,27 +67,27 @@ function patch_broken_eyaml { dependency = spec.dependencies.find { |d| d.name == "hiera-eyaml" } EOF - fi - fi - fi - - # - # Ubuntu 24.04 (noble) has a hiera-eyaml version that is incompatible with ruby 3.2+ (default in ubuntu24). - # This is fixed in hiera-eyaml version 3.3.0: https://github.com/voxpupuli/hiera-eyaml/pull/340/files - # https://github.com/voxpupuli/hiera-eyaml/blob/master/CHANGELOG.md - # But there is no modern version of hiera-eyaml packaged in debian or ubuntu. - # https://github.com/puppetlabs/puppet/wiki/Puppet-8-Compatibility#filedirexists-removed - # - - . /etc/os-release - if [ "${VERSION_CODENAME}" == "noble" ]; then - plugins_file="/usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb" - if [ -f $plugins_file ]; then - # We only want to try patching the file if it is the known broken version - bad_sum="59c6eb910ab2eb44f8c75aeaa79bff097038feb673b5c6bdccde23d9b2a393e2" - sum=$(sha256sum $plugins_file | awk '{print $1}') - if [ "$sum" == "$bad_sum" ]; then - patch --fuzz=0 --directory=/ --strip=0 <<'EOF' + fi + fi + fi + + # + # Ubuntu 24.04 (noble) has a hiera-eyaml version that is incompatible with ruby 3.2+ (default in ubuntu24). + # This is fixed in hiera-eyaml version 3.3.0: https://github.com/voxpupuli/hiera-eyaml/pull/340/files + # https://github.com/voxpupuli/hiera-eyaml/blob/master/CHANGELOG.md + # But there is no modern version of hiera-eyaml packaged in debian or ubuntu. + # https://github.com/puppetlabs/puppet/wiki/Puppet-8-Compatibility#filedirexists-removed + # + + . /etc/os-release + if [ "${VERSION_CODENAME}" == "noble" ]; then + plugins_file="/usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb" + if [ -f $plugins_file ]; then + # We only want to try patching the file if it is the known broken version + bad_sum="59c6eb910ab2eb44f8c75aeaa79bff097038feb673b5c6bdccde23d9b2a393e2" + sum=$(sha256sum $plugins_file | awk '{print $1}') + if [ "$sum" == "$bad_sum" ]; then + patch --fuzz=0 --directory=/ --strip=0 <<'EOF' --- /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb.orig 2022-06-11 16:30:10.000000000 +0000 +++ /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb 2024-09-09 14:13:19.306342025 +0000 @@ -59,7 +59,7 @@ @@ -101,229 +100,227 @@ EOF options[:input_data] = File.read options[:eyaml] rescue EOF - fi - fi - fi -} - -function edit_copy_and_commit() -{ - # - # This code runs on the administrators local machine - # - local host=$1 - local global=$2 - - if [[ ${EDITOR} ]]; then - declare -r REMOTE_EDITOR="${EDITOR}" - else - declare -r REMOTE_EDITOR='/usr/bin/vim.tiny' - fi - - # Execute this script, on a remote host - ssh -t root@"${host}" EDITOR="${REMOTE_EDITOR}" /var/cache/cosmos/repo/edit-secrets --on-host "$global" - scp -q root@"${host}:${LAST_OUTPUT_FILENAME}" "${TMPFILE}" - - local save_to - if grep ^"STATUS=UPDATED" "$TMPFILE" > /dev/null; then - save_to="${host}/overlay/etc/hiera/data/secrets.yaml.asc" - - # extract the GPG output - perl -e '$a = 0; while (<>) { $a = 1 if ($_ =~ /-+BEGIN PGP MESSAGE-+/); - print $_ if $a; $a = 0 if ($_ =~ /-+END PGP MESSAGE-+/); }' < "$TMPFILE" > "$TMPFILE2" - - if ! grep "END PGP MESSAGE" "$TMPFILE2" > /dev/null; then - echo "$0: Failed extracting PGP output from file $TMPFILE into $TMPFILE2" - exit 1 - fi - elif grep ^"STATUS=EYAML_UPDATED" "$TMPFILE" > /dev/null; then - if [ -n "$global" ]; then - save_to="${global}/overlay/etc/hiera/data/shared-cosmos.eyaml" - else - save_to="${host}/overlay/etc/hiera/data/local.eyaml" - fi - - # extract the eyaml output - perl -e '$a = 0; while (<>) { $a = 1 if ($_ =~ /^---$/); - print $_ if $a }' < "$TMPFILE" > "$TMPFILE2" - - if ! grep "^---$" "$TMPFILE2" > /dev/null; then - echo "$0: Failed extracting yaml output from file $TMPFILE into $TMPFILE2" - exit 1 + fi + fi fi - else - echo "" - echo "Not updated" - echo "" +} - exit 0 - fi +function edit_copy_and_commit() { + # + # This code runs on the administrators local machine + # + local host=$1 + local global=$2 - # use cat to preserve permissions etc. - mkdir -p "$(dirname "${save_to}")" - cat "$TMPFILE2" > "${save_to}" - git add "${save_to}" + if [[ ${EDITOR} ]]; then + declare -r REMOTE_EDITOR="${EDITOR}" + else + declare -r REMOTE_EDITOR='/usr/bin/vim.tiny' + fi - if grep ^"STATUS=EYAML_UPDATED" "$TMPFILE" > /dev/null; then - git diff --cached "${save_to}" - fi + # Execute this script, on a remote host + ssh -t root@"${host}" EDITOR="${REMOTE_EDITOR}" /var/cache/cosmos/repo/edit-secrets --on-host "$global" + scp -q root@"${host}:${LAST_OUTPUT_FILENAME}" "${TMPFILE}" + + local save_to + if grep ^"STATUS=UPDATED" "$TMPFILE" >/dev/null; then + save_to="${host}/overlay/etc/hiera/data/secrets.yaml.asc" + + # extract the GPG output + perl -e '$a = 0; while (<>) { $a = 1 if ($_ =~ /-+BEGIN PGP MESSAGE-+/); + print $_ if $a; $a = 0 if ($_ =~ /-+END PGP MESSAGE-+/); }' <"$TMPFILE" >"$TMPFILE2" + + if ! grep "END PGP MESSAGE" "$TMPFILE2" >/dev/null; then + echo "$0: Failed extracting PGP output from file $TMPFILE into $TMPFILE2" + exit 1 + fi + elif grep ^"STATUS=EYAML_UPDATED" "$TMPFILE" >/dev/null; then + if [ -n "$global" ]; then + save_to="${global}/overlay/etc/hiera/data/shared-cosmos.eyaml" + else + save_to="${host}/overlay/etc/hiera/data/local.eyaml" + fi + + # extract the eyaml output + perl -e '$a = 0; while (<>) { $a = 1 if ($_ =~ /^---$/); + print $_ if $a }' <"$TMPFILE" >"$TMPFILE2" + + if ! grep "^---$" "$TMPFILE2" >/dev/null; then + echo "$0: Failed extracting yaml output from file $TMPFILE into $TMPFILE2" + exit 1 + fi + else + echo "" + echo "Not updated" + echo "" - echo "" - echo "$save_to updated" - echo "" + exit 0 + fi - exit 0 -} + # use cat to preserve permissions etc. + mkdir -p "$(dirname "${save_to}")" + cat "$TMPFILE2" >"${save_to}" + git add "${save_to}" -function edit_file_on_host() { - # - # Local execution on a host - # - - - if [ -n "$1" ]; then - local EYAMLFILE=/etc/hiera/data/shared-cosmos.eyaml - local EYAMLKEY=/etc/hiera/eyaml/shared-cosmos.pkcs7.key - local EYAMLPUBKEY=/etc/hiera/eyaml/shared-cosmos.pkcs7.pem - local SECRETFILE=/etc/hiera/data/shared-secrets.yaml.asc - else - local EYAMLFILE=/etc/hiera/data/local.eyaml - local EYAMLKEY=/etc/hiera/eyaml/private_key.pkcs7.pem - local EYAMLPUBKEY=/etc/hiera/eyaml/public_certkey.pkcs7.pem - local SECRETFILE=/etc/hiera/data/secrets.yaml.asc - fi - - if [ -f "${EYAMLFILE}" ]; then - edit_eyaml_file ${EYAMLFILE} ${EYAMLKEY} ${EYAMLPUBKEY} - elif [ -f "${SECRETFILE}" ]; then - edit_gpg_file ${SECRETFILE} - elif [ -f "${EYAMLKEY}" ]; then - # default to eyaml if the key exists and none of the secrets-file above exist - echo "---" > ${EYAMLFILE} - edit_eyaml_file ${EYAMLFILE} ${EYAMLKEY} ${EYAMLPUBKEY} - fi -} + if grep ^"STATUS=EYAML_UPDATED" "$TMPFILE" >/dev/null; then + git diff --cached "${save_to}" + fi -function edit_gpg_file() -{ - local SECRETFILE=$1 - - GNUPGHOME=/etc/hiera/gpg/ - export GNUPGHOME - - local GPG - GPG=$(which gpg2 || true) - if [ ! -x "$GPG" ]; then - GPG=$(which gpg || true) - if [ ! -x "$GPG" ]; then - echo "$0: gpg2 or gpg not found" - exit 1 - fi - fi - - if ! $GPG --list-secret-keys | grep -q ^"sec\s"; then - echo "$0: Secret key does not exist (in $GNUPGHOME)." echo "" - echo "Generate it with /var/cache/cosmos/model/pre-tasks.d/040hiera-gpg" + echo "$save_to updated" echo "" - exit 1 - fi - - if [ -s "$SECRETFILE" ]; then - $GPG -d "$SECRETFILE" > "$TMPFILE" - fi - - cp "$TMPFILE" "$TMPFILE2" - sensible-editor "$TMPFILE" - rm -f "${TMPFILE}"~ "${TMPFILE2}"~ - echo "" - echo "" + exit 0 +} - local status=0 - cmp -s "$TMPFILE" "$TMPFILE2" || status=1 - if [ $status -eq 0 ]; then - ( - echo "STATUS=NOT_CHANGED" - ) > $LAST_OUTPUT_FILENAME - echo "" - echo "$0: No changes detected" - else - # figure out this hosts gpg key id - if lsb_release -r | grep -qE '(18|20).04'; then - recipient=$($GPG --list-secret-keys | grep -A1 '^sec' | tail -1 | awk '{print $1}') +function edit_file_on_host() { + # + # Local execution on a host + # + + if [ -n "$1" ]; then + local EYAMLFILE=/etc/hiera/data/shared-cosmos.eyaml + local EYAMLKEY=/etc/hiera/eyaml/shared-cosmos.pkcs7.key + local EYAMLPUBKEY=/etc/hiera/eyaml/shared-cosmos.pkcs7.pem + local SECRETFILE=/etc/hiera/data/shared-secrets.yaml.asc else - recipient=$($GPG --list-secret-key | grep ^sec | head -1 | awk '{print $2}' | cut -d / -f 2) + local EYAMLFILE=/etc/hiera/data/local.eyaml + local EYAMLKEY=/etc/hiera/eyaml/private_key.pkcs7.pem + local EYAMLPUBKEY=/etc/hiera/eyaml/public_certkey.pkcs7.pem + local SECRETFILE=/etc/hiera/data/secrets.yaml.asc fi - save_to="$(hostname --fqdn)/overlay${SECRETFILE}" - echo "" - ( - echo "STATUS=UPDATED" - echo "" - ) > $LAST_OUTPUT_FILENAME - $GPG --output - --armor --recipient "$recipient" --sign --encrypt "$TMPFILE" >> $LAST_OUTPUT_FILENAME - echo "" - echo "GPG output saved in $LAST_OUTPUT_FILENAME - save it in Cosmos as" - echo "" - echo " $save_to" - echo "" - fi + if [ -f "${EYAMLFILE}" ]; then + edit_eyaml_file ${EYAMLFILE} ${EYAMLKEY} ${EYAMLPUBKEY} + elif [ -f "${SECRETFILE}" ]; then + edit_gpg_file ${SECRETFILE} + elif [ -f "${EYAMLKEY}" ]; then + # default to eyaml if the key exists and none of the secrets-file above exist + echo "---" >${EYAMLFILE} + edit_eyaml_file ${EYAMLFILE} ${EYAMLKEY} ${EYAMLPUBKEY} + fi } -function edit_eyaml_file() -{ - local EYAMLFILE=$1 +function edit_gpg_file() { + local SECRETFILE=$1 + + GNUPGHOME=/etc/hiera/gpg/ + export GNUPGHOME + + local GPG + GPG=$(which gpg2 || true) + if [ ! -x "$GPG" ]; then + GPG=$(which gpg || true) + if [ ! -x "$GPG" ]; then + echo "$0: gpg2 or gpg not found" + exit 1 + fi + fi - local privkey=$2 - local pubkey=$3 - for f in $privkey $pubkey; do - test -f "${f}" || { echo "$0: eyaml key file ${f} not found"; exit 1; } - done + if ! $GPG --list-secret-keys | grep -q ^"sec\s"; then + echo "$0: Secret key does not exist (in $GNUPGHOME)." + echo "" + echo "Generate it with /var/cache/cosmos/model/pre-tasks.d/040hiera-gpg" + echo "" + exit 1 + fi - patch_broken_eyaml + if [ -s "$SECRETFILE" ]; then + $GPG -d "$SECRETFILE" >"$TMPFILE" + fi - # save source file for comparision afterwards - cp "${EYAMLFILE}" "${TMPFILE}" - eyaml edit --pkcs7-private-key "${privkey}" --pkcs7-public-key "${pubkey}" "${EYAMLFILE}" + cp "$TMPFILE" "$TMPFILE2" + sensible-editor "$TMPFILE" + rm -f "${TMPFILE}"~ "${TMPFILE2}"~ - local status=0 - cmp -s "${EYAMLFILE}" "$TMPFILE" || status=1 - if [ $status -eq 0 ]; then - ( - echo "STATUS=NOT_CHANGED" - ) > $LAST_OUTPUT_FILENAME echo "" - echo "$0: No changes detected" - else echo "" - ( - echo "STATUS=EYAML_UPDATED" - echo "" - ) > $LAST_OUTPUT_FILENAME - cat "${EYAMLFILE}" >> $LAST_OUTPUT_FILENAME - fi + + local status=0 + cmp -s "$TMPFILE" "$TMPFILE2" || status=1 + if [ $status -eq 0 ]; then + ( + echo "STATUS=NOT_CHANGED" + ) >$LAST_OUTPUT_FILENAME + echo "" + echo "$0: No changes detected" + else + # figure out this hosts gpg key id + if lsb_release -r | grep -qE '(18|20).04'; then + recipient=$($GPG --list-secret-keys | grep -A1 '^sec' | tail -1 | awk '{print $1}') + else + recipient=$($GPG --list-secret-key | grep ^sec | head -1 | awk '{print $2}' | cut -d / -f 2) + fi + + save_to="$(hostname --fqdn)/overlay${SECRETFILE}" + echo "" + ( + echo "STATUS=UPDATED" + echo "" + ) >$LAST_OUTPUT_FILENAME + $GPG --output - --armor --recipient "$recipient" --sign --encrypt "$TMPFILE" >>$LAST_OUTPUT_FILENAME + echo "" + echo "GPG output saved in $LAST_OUTPUT_FILENAME - save it in Cosmos as" + echo "" + echo " $save_to" + echo "" + fi } +function edit_eyaml_file() { + local EYAMLFILE=$1 + + local privkey=$2 + local pubkey=$3 + for f in $privkey $pubkey; do + test -f "${f}" || { + echo "$0: eyaml key file ${f} not found" + exit 1 + } + done + + patch_broken_eyaml + + # save source file for comparision afterwards + cp "${EYAMLFILE}" "${TMPFILE}" + eyaml edit --pkcs7-private-key "${privkey}" --pkcs7-public-key "${pubkey}" "${EYAMLFILE}" + + local status=0 + cmp -s "${EYAMLFILE}" "$TMPFILE" || status=1 + if [ $status -eq 0 ]; then + ( + echo "STATUS=NOT_CHANGED" + ) >$LAST_OUTPUT_FILENAME + echo "" + echo "$0: No changes detected" + else + echo "" + ( + echo "STATUS=EYAML_UPDATED" + echo "" + ) >$LAST_OUTPUT_FILENAME + cat "${EYAMLFILE}" >>$LAST_OUTPUT_FILENAME + fi +} if [[ $1 == '--on-host' ]]; then - edit_file_on_host "$2" + edit_file_on_host "$2" else - host=${1/%\/*/} # remove trailing slashes - global=false - if [ -n "$2" ]; then - if [ ! -d "$2/overlay/etc/hiera/data/" ]; then - echo "Overlay folder is not configured with hiera $2/overlay/etc/hiera/data/" - exit 2 - fi - fi - - if [ ! -d "$host" ]; then - echo "$0: No host-directory for '$host' found - execute in top-level cosmos dir" - exit 1 - fi + host=${1/%\/*/} # remove trailing slashes + global=false + if [ -n "$2" ]; then + if [ ! -d "$2/overlay/etc/hiera/data/" ]; then + echo "Overlay folder is not configured with hiera $2/overlay/etc/hiera/data/" + exit 2 + fi + fi + + if [ ! -d "$host" ]; then + echo "$0: No host-directory for '$host' found - execute in top-level cosmos dir" + exit 1 + fi - edit_copy_and_commit "$host" "$2" + edit_copy_and_commit "$host" "$2" fi exit 0 diff --git a/global/post-tasks.d/010cosmos-modules b/global/post-tasks.d/010cosmos-modules index 092815a..4708fd1 100755 --- a/global/post-tasks.d/010cosmos-modules +++ b/global/post-tasks.d/010cosmos-modules @@ -12,23 +12,23 @@ set -e if [ -f "${COSMOS_MODEL}/overlay/etc/puppet/cosmos-modules.conf" ]; then - test "$COSMOS_VERBOSE" = "y" && \ - echo "$0: /etc/puppet/cosmos-modules.conf is present in the model, exiting" - exit 0 + test "$COSMOS_VERBOSE" = "y" && + echo "$0: /etc/puppet/cosmos-modules.conf is present in the model, exiting" + exit 0 fi if [ -x /etc/puppet/setup_cosmos_modules ]; then - test "$COSMOS_VERBOSE" = "y" && \ - echo "$0: Updating /etc/puppet/cosmos-modules.conf with /etc/puppet/setup_cosmos_modules" - /etc/puppet/setup_cosmos_modules + test "$COSMOS_VERBOSE" = "y" && + echo "$0: Updating /etc/puppet/cosmos-modules.conf with /etc/puppet/setup_cosmos_modules" + /etc/puppet/setup_cosmos_modules - test -f /etc/puppet/cosmos-modules.conf && exit 0 + test -f /etc/puppet/cosmos-modules.conf && exit 0 fi -test "$COSMOS_VERBOSE" = "y" && \ - echo "$0: Creating/updating /etc/puppet/cosmos-modules.conf with defaults from this script" +test "$COSMOS_VERBOSE" = "y" && + echo "$0: Creating/updating /etc/puppet/cosmos-modules.conf with defaults from this script" -cat > /etc/puppet/cosmos-modules.conf << EOF +cat >/etc/puppet/cosmos-modules.conf <&1) - # Only print output if a key is changed - echo "$gpg_output" | grep -q " not changed$" || echo "$gpg_output" + # Always import a non-expired file since it may have been updated + gpg_output=$(cosmos gpg --no-tty --import <"$k" 2>&1) + # Only print output if a key is changed + echo "$gpg_output" | grep -q " not changed$" || echo "$gpg_output" done # Load information about all keys present in the GPG keyring for fp in $(cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" { print $5 }'); do - KEYRING[$fp]='1' + KEYRING[$fp]='1' done -if (( ${#SEEN[@]} == 0 )); then - echo -e "$0: ${red}NO trusted keys found in directory ${COSMOS_KEYS} - aborting${reset}" - echo "(this is probably a syntax problem with the gpg commands in this script)" - exit 1 +if ((${#SEEN[@]} == 0)); then + echo -e "$0: ${red}NO trusted keys found in directory ${COSMOS_KEYS} - aborting${reset}" + echo "(this is probably a syntax problem with the gpg commands in this script)" + exit 1 fi # Delete keys no longer present (or expired) in $COSMOS_KEYS directory for fp in "${!KEYRING[@]}"; do - if [[ ! ${SEEN[$fp]} ]]; then - echo -e "$0: ${bold}Deleting key${reset} ${fp} not present (or expired) in ${COSMOS_KEYS}" - cosmos gpg --fingerprint "$fp" - cosmos gpg --yes --batch --delete-key "$fp" || true - fi + if [[ ! ${SEEN[$fp]} ]]; then + echo -e "$0: ${bold}Deleting key${reset} ${fp} not present (or expired) in ${COSMOS_KEYS}" + cosmos gpg --fingerprint "$fp" + cosmos gpg --yes --batch --delete-key "$fp" || true + fi done diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index 39569b2..e223326 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -14,99 +14,105 @@ reset='\e[0m' red='\033[01;31m' stage_module() { - rm -rf $CACHE_DIR/staging/$1 - git archive --format=tar --prefix=$1/ $2 | (cd $CACHE_DIR/staging/ && tar xf -) + rm -rf "$CACHE_DIR/staging/$1" + git archive --format=tar --prefix="$1"/ "$2" | (cd "$CACHE_DIR/staging/" && tar xf -) } -if [ -f $CONFIG -o $LOCALCONFIG ]; then - if [ ! -d $MODULES_DIR ]; then - mkdir -p $MODULES_DIR - fi - if [ ! -d $CACHE_DIR ]; then - mkdir -p $CACHE_DIR/{scm,staging} - fi +if [ -f "$CONFIG" -o "$LOCALCONFIG" ]; then + if [ ! -d "$MODULES_DIR" ]; then + mkdir -p "$MODULES_DIR" + fi + if [ ! -d "$CACHE_DIR" ]; then + mkdir -p "$CACHE_DIR"/{scm,staging} + fi - test -f $CONFIG || CONFIG='' - test -f $LOCALCONFIG || LOCALCONFIG='' + files=() + if [ -f "$CONFIG" ]; then + files+=("$CONFIG") + fi - # First pass to clone any new modules, and update those marked for updating. - grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( - while read module src update pattern; do - # We only support git://, file:/// and https:// urls at the moment - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then - if [ ! -d $CACHE_DIR/scm/$module ]; then - git clone -q $src $CACHE_DIR/scm/$module - elif [ -d $CACHE_DIR/scm/$module/.git ]; then - if [ "$update" = "yes" ]; then - cd $CACHE_DIR/scm/$module - if [ "$src" != "$(git config remote.origin.url)" ]; then - git config remote.origin.url $src - fi - # Support master branch being renamed to main - git branch --all | grep -q '^[[:space:]]*remotes/origin/main$' && git checkout main - # Update repo and clean out any local inconsistencies - git pull -q || (git fetch && git reset --hard) - else - continue - fi - else - echo -e "${red}ERROR: Ignoring non-git repository${reset}" - continue - fi - elif [[ "$src" =~ .*:// ]]; then - echo -e "${red}ERROR: Don't know how to install '${src}'${reset}" - continue - else - echo -e "${bold}WARNING - attempting UNSAFE installation/upgrade of puppet-module ${module} from ${src}${reset}" - if [ ! -d /etc/puppet/modules/$module ]; then - puppet module install $src - elif [ "$update" = "yes" ]; then - puppet module upgrade $src - fi - fi - done - ) + if [ -f "$LOCALCONFIG" ]; then + files+=("$LOCALCONFIG") + fi - # Second pass to verify the signatures on all modules and stage those that - # have good signatures. - grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( - while read module src update pattern; do - # We only support git://, file:/// and https:// urls at the moment - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then - # Verify git tag - cd $CACHE_DIR/scm/$module - TAG=$(git tag -l "${pattern:-*}" | sort | tail -1) - if [ "$COSMOS_VERBOSE" = "y" ]; then - echo -e "Checking signature on puppet-module:tag ${bold}${module}:${TAG}${reset}" - fi - if [ -z "$TAG" ]; then - echo -e "${red}ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module ${module}${reset}" - continue - fi - git tag -v $TAG &> /dev/null - if [ $? == 0 ]; then - #if [ "$COSMOS_VERBOSE" = "y" ]; then - # # short output on good signature - # git tag -v $TAG 2>&1 | grep "gpg: Good signature" - #fi - # Put archive in staging since tag verified OK - stage_module $module $TAG - else - echo -e "${red}FAILED signature check on puppet-module ${module}${reset}" - git tag -v $TAG - echo '' - fi - fi - done - ) + # First pass to clone any new modules, and update those marked for updating. + grep -h -E -v "^#" "${files[@]}" | sort | ( + while read module src update pattern; do + # We only support git://, file:/// and https:// urls at the moment + if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then + if [ ! -d "$CACHE_DIR/scm/$module" ]; then + git clone -q "$src" "$CACHE_DIR/scm/$module" + elif [ -d "$CACHE_DIR/scm/$module/.git" ]; then + if [ "$update" = "yes" ]; then + cd "$CACHE_DIR/scm/$module" || exit 1 + if [ "$src" != "$(git config remote.origin.url)" ]; then + git config remote.origin.url "$src" + fi + # Support master branch being renamed to main + git branch --all | grep -q '^[[:space:]]*remotes/origin/main$' && git checkout main + # Update repo and clean out any local inconsistencies + git pull -q || (git fetch && git reset --hard) + else + continue + fi + else + echo -e "${red}ERROR: Ignoring non-git repository${reset}" + continue + fi + elif [[ "$src" =~ .*:// ]]; then + echo -e "${red}ERROR: Don't know how to install '${src}'${reset}" + continue + else + echo -e "${bold}WARNING - attempting UNSAFE installation/upgrade of puppet-module ${module} from ${src}${reset}" + if [ ! -d "/etc/puppet/modules/$module" ]; then + puppet module install "$src" + elif [ "$update" = "yes" ]; then + puppet module upgrade "$src" + fi + fi + done + ) - # Cleanup removed puppet modules from CACHE_DIR - for MODULE in $(ls -1 $CACHE_DIR/staging/); do - if ! grep -h -E -q "^$MODULE\s+" $CONFIG $LOCALCONFIG; then - rm -rf $CACHE_DIR/{scm,staging}/$MODULE - fi - done + # Second pass to verify the signatures on all modules and stage those that + # have good signatures. + grep -h -E -v "^#" "${files[@]}" | sort | ( + while read module src update pattern; do + # We only support git://, file:/// and https:// urls at the moment + if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then + # Verify git tag + cd "$CACHE_DIR/scm/$module" || exit 1 + TAG=$(git tag -l "${pattern:-*}" | sort | tail -1) + if [ "$COSMOS_VERBOSE" = "y" ]; then + echo -e "Checking signature on puppet-module:tag ${bold}${module}:${TAG}${reset}" + fi + if [ -z "$TAG" ]; then + echo -e "${red}ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module ${module}${reset}" + continue + fi + git tag -v "$TAG" &>/dev/null + if [ $? == 0 ]; then + #if [ "$COSMOS_VERBOSE" = "y" ]; thengg + # # short output on good signature + # git tag -v $TAG 2>&1 | grep "gpg: Good signature" + #fi + # Put archive in staging since tag verified OK + stage_module "$module" "$TAG" + else + echo -e "${red}FAILED signature check on puppet-module ${module}${reset}" + git tag -v "$TAG" + echo '' + fi + fi + done + ) - # Installing verified puppet modules - rsync --archive --delete $CACHE_DIR/staging/ $MODULES_DIR/ + # Cleanup removed puppet modules from CACHE_DIR + for MODULE in $(ls -1 $CACHE_DIR/staging/); do + if ! grep -h -E -q "^$MODULE\s+" "$CONFIG" "$LOCALCONFIG"; then + rm -rf "$CACHE_DIR"/{scm,staging}/"$MODULE" + fi + done + + # Installing verified puppet modules + rsync --archive --delete "$CACHE_DIR/staging/" "$MODULES_DIR/" fi diff --git a/global/post-tasks.d/030puppet b/global/post-tasks.d/030puppet index 39c0f33..5cc09c9 100755 --- a/global/post-tasks.d/030puppet +++ b/global/post-tasks.d/030puppet @@ -1,21 +1,23 @@ -#!/bin/sh +#!/bin/bash set -e +args=() if [ "x$COSMOS_VERBOSE" = "xy" ]; then - args="--verbose --show_diff" + args+=('--verbose') + args+=('--show_diff') else - args="--logdest=syslog" + args+=('--logdest=syslog') fi if [ -f /usr/bin/puppet ] && [ -d /etc/puppet/manifests ]; then - find /etc/puppet/manifests -name \*.pp | while read -r m; do - test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m" - puppet apply $args $m - done + find /etc/puppet/manifests -name \*.pp | while read -r m; do + test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m" + puppet apply "${args[@]}" "$m" + done - PUPPET_REPORTS_DIR='/var/lib/puppet/reports' - if [ -d "${PUPPET_REPORTS_DIR}" ]; then - find "${PUPPET_REPORTS_DIR}" -type f -mtime +10 -print0 | xargs -0 rm -f - fi + PUPPET_REPORTS_DIR='/var/lib/puppet/reports' + if [ -d "${PUPPET_REPORTS_DIR}" ]; then + find "${PUPPET_REPORTS_DIR}" -type f -mtime +10 -print0 | xargs -0 rm -f + fi fi diff --git a/global/post-tasks.d/099autoremove b/global/post-tasks.d/099autoremove index c3c809c..cc46a0f 100755 --- a/global/post-tasks.d/099autoremove +++ b/global/post-tasks.d/099autoremove @@ -2,7 +2,7 @@ export DEBIAN_FRONTEND='noninteractive' -if (( $RANDOM % 20 == 0)); then - apt-get -qq update - apt-get -qq -y autoremove +if (($RANDOM % 20 == 0)); then + apt-get -qq update + apt-get -qq -y autoremove fi diff --git a/global/post-tasks.d/999reboot b/global/post-tasks.d/999reboot index a83251e..17d37aa 100755 --- a/global/post-tasks.d/999reboot +++ b/global/post-tasks.d/999reboot @@ -2,39 +2,39 @@ if [[ -f /var/run/reboot-required && -f /etc/cosmos-automatic-reboot ]]; then + # May contain ALLOW_REBOOT_AT= + # Eg. ALLOW_REBOOT_AT=06 + # shellcheck source=/dev/null + . /etc/cosmos-automatic-reboot - # May contain ALLOW_REBOOT_AT= - # Eg. ALLOW_REBOOT_AT=06 - . /etc/cosmos-automatic-reboot + if [ -n "${ALLOW_REBOOT_AT}" ]; then + if [ "${ALLOW_REBOOT_AT}" != "$(date +%H)" ]; then + echo "Scheduled to reboot at ${ALLOW_REBOOT_AT}" + exit + fi + fi - if [ -n "${ALLOW_REBOOT_AT}" ]; then - if [ "${ALLOW_REBOOT_AT}" != "$(date +%H)" ]; then - echo "Scheduled to reboot at ${ALLOW_REBOOT_AT}" - exit - fi - fi + if [[ $HOSTNAME =~ -tug- ]]; then + # Reboot hosts in site TUG with 15 seconds delay (enough to manually + # cancel the reboot if logged in and seeing the 'emerg' message broadcasted to console) + sleep=15 + elif [[ $HOSTNAME =~ -fre- ]]; then + # reboot hosts in site FRE with 15+180 to 15+180+180 seconds delay + sleep=$((180 + ($RANDOM % 180))) + elif [[ $HOSTNAME =~ -lla- ]]; then + # reboot hosts in site LLA with 15+180+180 to 15+180+180+180 seconds delay + sleep=$((375 + ($RANDOM % 180))) + else + # reboot hosts in any other site with 15 to 315 seconds delay + sleep=$((15 + ($RANDOM % 300))) + fi - if [[ $HOSTNAME =~ -tug- ]]; then - # Reboot hosts in site TUG with 15 seconds delay (enough to manually - # cancel the reboot if logged in and seeing the 'emerg' message broadcasted to console) - sleep=15 - elif [[ $HOSTNAME =~ -fre- ]]; then - # reboot hosts in site FRE with 15+180 to 15+180+180 seconds delay - sleep=$(( 180 + ($RANDOM % 180))) - elif [[ $HOSTNAME =~ -lla- ]]; then - # reboot hosts in site LLA with 15+180+180 to 15+180+180+180 seconds delay - sleep=$(( 375 + ($RANDOM % 180))) - else - # reboot hosts in any other site with 15 to 315 seconds delay - sleep=$(( 15 + ($RANDOM % 300))) - fi - - logger -p local0.emerg -i -t cosmos-automatic-reboot "Rebooting automatically in $sleep seconds (if /var/run/reboot-required still exists)" - sleep $sleep - if [ -f /var/run/reboot-required ]; then - logger -p local0.crit -i -t cosmos-automatic-reboot "Rebooting automatically" - # Signal to run-cosmos - touch /var/run/cosmos-reboot-in-progress - reboot - fi + logger -p local0.emerg -i -t cosmos-automatic-reboot "Rebooting automatically in $sleep seconds (if /var/run/reboot-required still exists)" + sleep $sleep + if [ -f /var/run/reboot-required ]; then + logger -p local0.crit -i -t cosmos-automatic-reboot "Rebooting automatically" + # Signal to run-cosmos + touch /var/run/cosmos-reboot-in-progress + reboot + fi fi diff --git a/global/pre-tasks.d/014set-cosmos-permissions b/global/pre-tasks.d/014set-cosmos-permissions index 08992b5..e3a802c 100755 --- a/global/pre-tasks.d/014set-cosmos-permissions +++ b/global/pre-tasks.d/014set-cosmos-permissions @@ -11,13 +11,13 @@ set -e self=$(basename "$0") if ! test -d "$COSMOS_BASE"; then - test -z "$COSMOS_VERBOSE" || echo "$self: COSMOS_BASE was not found. Aborting change of permissions." - exit 0 + test -z "$COSMOS_VERBOSE" || echo "$self: COSMOS_BASE was not found. Aborting change of permissions." + exit 0 fi args="" if [ "x$COSMOS_VERBOSE" = "xy" ]; then - args="-v" + args="-v" fi chown ${args} root:root "$COSMOS_BASE" diff --git a/global/pre-tasks.d/015set-overlay-permissions b/global/pre-tasks.d/015set-overlay-permissions index 205180b..0fe89b7 100755 --- a/global/pre-tasks.d/015set-overlay-permissions +++ b/global/pre-tasks.d/015set-overlay-permissions @@ -10,21 +10,21 @@ self=$(basename "$0") MODEL_OVERLAY="$COSMOS_MODEL/overlay" if ! test -d "$MODEL_OVERLAY"; then - test -z "$COSMOS_VERBOSE" || echo "$self: overlay is a no-op" - exit 0 + test -z "$COSMOS_VERBOSE" || echo "$self: overlay is a no-op" + exit 0 fi args="" if [ "x$COSMOS_VERBOSE" = "xy" ]; then - args="-v" + args="-v" fi if [ -d "$MODEL_OVERLAY/root" ]; then - chown ${args} root:root "$MODEL_OVERLAY"/root - chmod ${args} 0700 "$MODEL_OVERLAY"/root + chown ${args} root:root "$MODEL_OVERLAY"/root + chmod ${args} 0700 "$MODEL_OVERLAY"/root fi if [ -d "$MODEL_OVERLAY/root/.ssh" ]; then - chown ${args} -R root:root "$MODEL_OVERLAY"/root/.ssh - chmod ${args} 0700 "$MODEL_OVERLAY"/root/.ssh + chown ${args} -R root:root "$MODEL_OVERLAY"/root/.ssh + chmod ${args} 0700 "$MODEL_OVERLAY"/root/.ssh fi diff --git a/global/pre-tasks.d/020common-tools b/global/pre-tasks.d/020common-tools index aece646..3da764b 100755 --- a/global/pre-tasks.d/020common-tools +++ b/global/pre-tasks.d/020common-tools @@ -7,10 +7,10 @@ set -e stamp="$COSMOS_BASE/stamps/common-tools-v01.stamp" -if ! test -f $stamp; then - apt-get -y install vim traceroute tcpdump molly-guard less rsync git-core unattended-upgrades - update-alternatives --set editor /usr/bin/vim.basic +if ! test -f "$stamp"; then + apt-get -y install vim traceroute tcpdump molly-guard less rsync git-core unattended-upgrades + update-alternatives --set editor /usr/bin/vim.basic - mkdir -p `dirname $stamp` - touch $stamp + mkdir -p "$(dirname "$stamp")" + touch "$stamp" fi diff --git a/global/pre-tasks.d/030puppet b/global/pre-tasks.d/030puppet index e6e5364..ed4b94e 100755 --- a/global/pre-tasks.d/030puppet +++ b/global/pre-tasks.d/030puppet @@ -8,24 +8,24 @@ set -e stamp="$COSMOS_BASE/stamps/puppet-tools-v01.stamp" if ! test -f "${stamp}" -a -f /usr/bin/puppet; then - apt-get update - apt-get -y install puppet - . /etc/os-release + apt-get update + apt-get -y install puppet + # shellcheck source=/dev/null + . /etc/os-release - # Note: in posix shell, string comparison is done with a single = - if [ "${ID}" = "debian" ] && [ "${VERSION_ID}" -ge 12 ] || ([ "${ID}" = "ubuntu" ] && $(dpkg --compare-versions ${VERSION_ID} ge 24.04)) ; then - apt-get -y install \ - cron \ - puppet-module-camptocamp-augeas \ - puppet-module-puppetlabs-apt \ - puppet-module-puppetlabs-concat \ - puppet-module-puppetlabs-cron-core \ - puppet-module-puppetlabs-stdlib \ - puppet-module-puppetlabs-vcsrepo + # Note: in posix shell, string comparison is done with a single = + if [ "${ID}" = "debian" ] && [ "${VERSION_ID}" -ge 12 ] || ([ "${ID}" = "ubuntu" ] && $(dpkg --compare-versions "${VERSION_ID}" ge 24.04)); then + apt-get -y install \ + cron \ + puppet-module-camptocamp-augeas \ + puppet-module-puppetlabs-apt \ + puppet-module-puppetlabs-concat \ + puppet-module-puppetlabs-cron-core \ + puppet-module-puppetlabs-stdlib \ + puppet-module-puppetlabs-vcsrepo - fi + fi - mkdir -p "$(dirname "${stamp}")" - touch "${stamp}" + mkdir -p "$(dirname "${stamp}")" + touch "${stamp}" fi - diff --git a/global/pre-tasks.d/040hiera-eyaml b/global/pre-tasks.d/040hiera-eyaml index 9656413..b2f80e8 100755 --- a/global/pre-tasks.d/040hiera-eyaml +++ b/global/pre-tasks.d/040hiera-eyaml @@ -18,21 +18,21 @@ stamp="$COSMOS_BASE/stamps/hiera-eyaml-v01.stamp" test -f "$stamp" && exit 0 if [ ! -f /usr/bin/eyaml ] || [ ! -d /usr/share/doc/yaml-mode ]; then - apt-get update - # If we don't install emacs before yaml-mode the default emacs package - # will be emacs-gtk which brings x11 with friends which we don't need. - apt-get -y install emacs-nox - apt-get -y install hiera-eyaml yaml-mode + apt-get update + # If we don't install emacs before yaml-mode the default emacs package + # will be emacs-gtk which brings x11 with friends which we don't need. + apt-get -y install emacs-nox + apt-get -y install hiera-eyaml yaml-mode fi if [ ! -f ${EYAMLDIR}/public_certkey.pkcs7.pem ] || [ ! -f ${EYAMLDIR}/private_key.pkcs7.pem ]; then - # hiera-eyaml wants a certificate and public key, not just a public key oddly enough - echo "$0: Generating eyaml key in ${EYAMLDIR} - this might take a while..." - mkdir -p /etc/hiera/eyaml - openssl req -x509 -newkey rsa:4096 -keyout ${EYAMLDIR}/private_key.pkcs7.pem \ - -out ${EYAMLDIR}/public_certkey.pkcs7.pem -days 3653 -nodes -sha256 \ - -subj "/C=SE/O=SUNET/OU=EYAML/CN=$(hostname)" - rm -f ${EYAMLDIR}/public_key.pkcs7.pem # cleanup + # hiera-eyaml wants a certificate and public key, not just a public key oddly enough + echo "$0: Generating eyaml key in ${EYAMLDIR} - this might take a while..." + mkdir -p /etc/hiera/eyaml + openssl req -x509 -newkey rsa:4096 -keyout ${EYAMLDIR}/private_key.pkcs7.pem \ + -out ${EYAMLDIR}/public_certkey.pkcs7.pem -days 3653 -nodes -sha256 \ + -subj "/C=SE/O=SUNET/OU=EYAML/CN=$(hostname)" + rm -f ${EYAMLDIR}/public_key.pkcs7.pem # cleanup fi mkdir -p "$(dirname "${stamp}")" diff --git a/global/pre-tasks.d/040hiera-gpg b/global/pre-tasks.d/040hiera-gpg index bc1da35..e90a840 100755 --- a/global/pre-tasks.d/040hiera-gpg +++ b/global/pre-tasks.d/040hiera-gpg @@ -20,31 +20,31 @@ stamp="$COSMOS_BASE/stamps/hiera-gpg-v01.stamp" test -f "$stamp" && exit 0 if [ ! -f /usr/lib/ruby/vendor_ruby/gpgme.rb ]; then - apt-get update - apt-get -y install ruby-gpgme + apt-get update + apt-get -y install ruby-gpgme fi if [ ! -s $GNUPGHOME/secring.gpg ]; then - if [ "x$1" != "x--force" ]; then - echo "" - echo "Automatic Hiera-GPG key generation DISABLED (to not block on missing entropy)" - echo "" - echo " Run \`$0 --force' manually" - echo "" - exit 0 - fi + if [ "x$1" != "x--force" ]; then + echo "" + echo "Automatic Hiera-GPG key generation DISABLED (to not block on missing entropy)" + echo "" + echo " Run \`$0 --force' manually" + echo "" + exit 0 + fi - if [ ! -f /usr/bin/gpg2 ]; then - apt-get update - apt-get -y install gnupg2 - fi + if [ ! -f /usr/bin/gpg2 ]; then + apt-get update + apt-get -y install gnupg2 + fi - mkdir -p $GNUPGHOME - chmod 700 $GNUPGHOME + mkdir -p $GNUPGHOME + chmod 700 $GNUPGHOME - TMPFILE=$(mktemp /tmp/hiera-gpg.XXXXXX) - cat > "$TMPFILE" <"$TMPFILE" < Date: Tue, 25 Nov 2025 15:21:44 +0100 Subject: [PATCH 02/36] Shellcheck and shfmt --- addhost | 15 ++++++++++----- edit-secrets | 3 +-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/addhost b/addhost index 3645b72..b5e00c2 100755 --- a/addhost +++ b/addhost @@ -45,10 +45,13 @@ if test -z "$cmd_hostname"; then exit 1 fi +proxyjump=() if [[ -n $cmd_proxy ]]; then - proxyjump="-o ProxyJump=${cmd_proxy}" + proxyjump+=("-o") + proxyjump+=("ProxyJump=${cmd_proxy}i") fi +# shellcheck source=/dev/null test -f cosmos.conf && . ./cosmos.conf _remote=${remote:='ro'} @@ -70,8 +73,10 @@ fi if [ "$cmd_do_bootstrap" = "yes" ]; then cosmos_deb=$(find apt/ -maxdepth 1 -name 'cosmos_*.deb' | sort -V | tail -1) - scp $proxyjump "$cosmos_deb" apt/bootstrap-cosmos.sh root@"$cmd_hostname": - ssh root@"$cmd_hostname" $proxyjump ./bootstrap-cosmos.sh "$cmd_fqdn" "$rrepo" "$rtag" - ssh root@"$cmd_hostname" $proxyjump cosmos update - ssh root@"$cmd_hostname" $proxyjump cosmos apply + scp "${proxyjump[@]}" "$cosmos_deb" apt/bootstrap-cosmos.sh root@"$cmd_hostname": + # We want to expand the variables on clientside - shellcheck ignore + # shellcheck disable=SC2029 + ssh root@"$cmd_hostname" "${proxyjump[@]}" ./bootstrap-cosmos.sh "$cmd_fqdn" "$rrepo" "$rtag" + ssh root@"$cmd_hostname" "${proxyjump[@]}" cosmos update + ssh root@"$cmd_hostname" "${proxyjump[@]}" cosmos apply fi diff --git a/edit-secrets b/edit-secrets index 1938726..2e059da 100755 --- a/edit-secrets +++ b/edit-secrets @@ -79,8 +79,7 @@ EOF # https://github.com/puppetlabs/puppet/wiki/Puppet-8-Compatibility#filedirexists-removed # - . /etc/os-release - if [ "${VERSION_CODENAME}" == "noble" ]; then + if [ "$(lsb_release -cs)" == "noble" ]; then plugins_file="/usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb" if [ -f $plugins_file ]; then # We only want to try patching the file if it is the known broken version From aaccee17c4eba14508f765b6ae1e05ff2d7394b1 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:23:56 +0100 Subject: [PATCH 03/36] shellcheck and shfmt --- scripts/test-in-docker.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/test-in-docker.sh b/scripts/test-in-docker.sh index dae44e1..959264d 100755 --- a/scripts/test-in-docker.sh +++ b/scripts/test-in-docker.sh @@ -17,13 +17,15 @@ test -d /var/cache/cosmos/model || mkdir -p /var/cache/cosmos/model # Make every "cosmos update" copy the contents from /multiverse # without requiring the changes in there to be checked into git. -cat >/etc/cosmos/update.d/50update-while-testing << EOF +cat >/etc/cosmos/update.d/50update-while-testing < Date: Tue, 25 Nov 2025 15:26:20 +0100 Subject: [PATCH 04/36] shellcheck and shfmt --- prepare-iaas-ubuntu | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/prepare-iaas-ubuntu b/prepare-iaas-ubuntu index 4a8ad39..785e6e0 100755 --- a/prepare-iaas-ubuntu +++ b/prepare-iaas-ubuntu @@ -3,16 +3,18 @@ ip="${1}" ssh_proxy="${2}" if [[ -z "${ip}" ]]; then - echo "Please specify a cloud image host that the script should do the following on:" - echo " #1 enable root-login" - echo " #2 remove the default user" - echo " #3 run apt-get update and dist-upgrade without interaction" - echo " #4 reboot to start using the new kernel, updated packages etc." - exit 1 + echo "Please specify a cloud image host that the script should do the following on:" + echo " #1 enable root-login" + echo " #2 remove the default user" + echo " #3 run apt-get update and dist-upgrade without interaction" + echo " #4 reboot to start using the new kernel, updated packages etc." + exit 1 fi +proxyjump=() if [[ -n "${ssh_proxy}" ]]; then - proxyjump="-o ProxyJump=${ssh_proxy}" + proxyjump+=("-o") + proxyjump+=("ProxyJump=${ssh_proxy}") fi set -x @@ -25,5 +27,5 @@ script_dir=$(dirname "$0") # === # userdel: user ubuntu is currently used by process 44063 # === -ssh "ubuntu@${ip}" ${proxyjump} "bash -s" < "$script_dir"/iaas-enable-root.sh -ssh "root@${ip}" ${proxyjump} "bash -s" < "$script_dir"/iaas-setup.sh +ssh "ubuntu@${ip}" "${proxyjump[@]}" "bash -s" <"$script_dir"/iaas-enable-root.sh +ssh "root@${ip}" "${proxyjump[@]}" "bash -s" <"$script_dir"/iaas-setup.sh From 3cb5e46fea9d7eec9310d39fa7033ff5978f41cd Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:27:23 +0100 Subject: [PATCH 05/36] shellcheck and shfmt --- prepare-iaas-debian | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/prepare-iaas-debian b/prepare-iaas-debian index 30c395a..2e9c31d 100755 --- a/prepare-iaas-debian +++ b/prepare-iaas-debian @@ -3,15 +3,17 @@ ip="${1}" ssh_proxy="${2}" if [[ -z "${ip}" ]]; then - echo "Please specify a cloud image host that the script should do the following on:" - echo " #1 enable root-login" - echo " #2 remove the default user" - echo " #3 run apt-get update and dist-upgrade without interaction" - echo " #4 reboot to start using the new kernel, updated packages etc." - exit 1 + echo "Please specify a cloud image host that the script should do the following on:" + echo " #1 enable root-login" + echo " #2 remove the default user" + echo " #3 run apt-get update and dist-upgrade without interaction" + echo " #4 reboot to start using the new kernel, updated packages etc." + exit 1 fi +proxyjump=() if [[ -n "${ssh_proxy}" ]]; then - proxyjump="-o ProxyJump=${ssh_proxy}" + proxyjump+=("-o") + proxyjump+=("ProxyJump=${ssh_proxy}") fi set -x @@ -25,5 +27,5 @@ script_dir=$(dirname "$0") # === # userdel: user debian is currently used by process 1082 # === -ssh "debian@${ip}" ${proxyjump} "bash -s" < "$script_dir"/iaas-enable-root.sh -ssh "root@${ip}" ${proxyjump} "bash -s" < "$script_dir"/iaas-setup.sh +ssh "debian@${ip}" "${proxyjump[@]}" "bash -s" <"$script_dir"/iaas-enable-root.sh +ssh "root@${ip}" "${proxyjump[@]}" "bash -s" <"$script_dir"/iaas-setup.sh From 7b479df74516a9f4ed0f4215a8ddcb4824d87362 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:29:45 +0100 Subject: [PATCH 06/36] shellcheck and shfmt --- host-puppet-conf-test | 48 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/host-puppet-conf-test b/host-puppet-conf-test index 2ddcbe9..753dc8c 100755 --- a/host-puppet-conf-test +++ b/host-puppet-conf-test @@ -1,43 +1,41 @@ #!/bin/bash set +x HOSTNAME=$1 -PUPPET_ARGS=$2 +PUPPET_ARGS=() +PUPPET_ARGS+=("$2") +PUPPET_ARGS+=("--verbose") if [ -z "$HOSTNAME" ]; then - echo "Usage: $0 fqdn" - exit 1 + echo "Usage: $0 fqdn" + exit 1 fi if [ ! -d "$HOSTNAME" ]; then - echo "$0: No host-directory for '$HOSTNAME' found - execute in top-level cosmos dir" - exit 1 + echo "$0: No host-directory for '$HOSTNAME' found - execute in top-level cosmos dir" + exit 1 fi -PUPPET_ARGS=${PUPPET_ARGS-"--verbose"} - # Check if cosmos or puppet is already running on host echo "Checking if puppet or cosmos is already running..." ssh root@"$HOSTNAME" ps aux | grep -Ev "grep|edit-secrets|gpg-agent" | grep -Eq "cosmos|puppet" -if [ $? -eq 1 ] -then - echo "Copying files to host..." - rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-rules.yaml root@"$HOSTNAME":/etc/puppet/cosmos-rules.yaml - rsync -av --exclude '*~' global/overlay/etc/puppet/manifests/cosmos-site.pp root@"$HOSTNAME":/etc/puppet/manifests/cosmos-site.pp - rsync -av --exclude '*~' global/overlay/etc/hiera/data/common.yaml root@"$HOSTNAME":/etc/hiera/data/common.yaml +if [ $? -eq 1 ]; then + echo "Copying files to host..." + rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-rules.yaml root@"$HOSTNAME":/etc/puppet/cosmos-rules.yaml + rsync -av --exclude '*~' global/overlay/etc/puppet/manifests/cosmos-site.pp root@"$HOSTNAME":/etc/puppet/manifests/cosmos-site.pp + rsync -av --exclude '*~' global/overlay/etc/hiera/data/common.yaml root@"$HOSTNAME":/etc/hiera/data/common.yaml - # Test if the user has symlinked puppet-sunet correctly - # by first checking if the link exits and then whether - # or not the directory contains any files. - if [ -L global/overlay/etc/puppet/cosmos-modules/sunet ] && \ - [ -n "$(ls -A global/overlay/etc/puppet/cosmos-modules/sunet/*)" ] - then - rsync -av --delete --exclude '*~' global/overlay/etc/puppet/cosmos-modules/sunet/* root@$HOSTNAME:/etc/puppet/cosmos-modules/sunet/. - fi + # Test if the user has symlinked puppet-sunet correctly + # by first checking if the link exits and then whether + # or not the directory contains any files. + if [ -L global/overlay/etc/puppet/cosmos-modules/sunet ] && + [ -n "$(ls -A global/overlay/etc/puppet/cosmos-modules/sunet/*)" ]; then + rsync -av --delete --exclude '*~' global/overlay/etc/puppet/cosmos-modules/sunet/* root@"$HOSTNAME":/etc/puppet/cosmos-modules/sunet/. + fi - echo "Running puppet apply..." - ssh root@"$HOSTNAME" /usr/bin/puppet apply $PUPPET_ARGS /etc/puppet/manifests/cosmos-site.pp + echo "Running puppet apply..." + ssh root@"$HOSTNAME" /usr/bin/puppet apply "${PUPPET_ARGS[@]}" /etc/puppet/manifests/cosmos-site.pp else - echo "Cosmos or puppet already running. Exiting." - exit 1 + echo "Cosmos or puppet already running. Exiting." + exit 1 fi From 394377432864f71fa9edb3f6ac9b3100b6488b61 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:32:52 +0100 Subject: [PATCH 07/36] shellcheck and shfmt --- global/pre-tasks.d/040hiera-gpg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/pre-tasks.d/040hiera-gpg b/global/pre-tasks.d/040hiera-gpg index e90a840..6877518 100755 --- a/global/pre-tasks.d/040hiera-gpg +++ b/global/pre-tasks.d/040hiera-gpg @@ -26,7 +26,7 @@ fi if [ ! -s $GNUPGHOME/secring.gpg ]; then - if [ "x$1" != "x--force" ]; then + if [ "$1" != "--force" ]; then echo "" echo "Automatic Hiera-GPG key generation DISABLED (to not block on missing entropy)" echo "" From 9bc77dfff417cadb3f9d6916dabc6759ef0efc41 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:39:50 +0100 Subject: [PATCH 08/36] shellcheck and shfmt --- global/pre-tasks.d/030puppet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/pre-tasks.d/030puppet b/global/pre-tasks.d/030puppet index ed4b94e..7772091 100755 --- a/global/pre-tasks.d/030puppet +++ b/global/pre-tasks.d/030puppet @@ -14,7 +14,7 @@ if ! test -f "${stamp}" -a -f /usr/bin/puppet; then . /etc/os-release # Note: in posix shell, string comparison is done with a single = - if [ "${ID}" = "debian" ] && [ "${VERSION_ID}" -ge 12 ] || ([ "${ID}" = "ubuntu" ] && $(dpkg --compare-versions "${VERSION_ID}" ge 24.04)); then + if [ "${ID}" = "debian" ] && [ "${VERSION_ID}" -ge 12 ] || ([ "${ID}" = "ubuntu" ] && dpkg --compare-versions "${VERSION_ID}" ge 24.04); then apt-get -y install \ cron \ puppet-module-camptocamp-augeas \ From d7d6afa749a2c73c555262dc58a3a48dfcc1d51a Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:40:41 +0100 Subject: [PATCH 09/36] shellcheck and shfmt --- global/pre-tasks.d/015set-overlay-permissions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/pre-tasks.d/015set-overlay-permissions b/global/pre-tasks.d/015set-overlay-permissions index 0fe89b7..2a175be 100755 --- a/global/pre-tasks.d/015set-overlay-permissions +++ b/global/pre-tasks.d/015set-overlay-permissions @@ -15,7 +15,7 @@ if ! test -d "$MODEL_OVERLAY"; then fi args="" -if [ "x$COSMOS_VERBOSE" = "xy" ]; then +if [ "$COSMOS_VERBOSE" = "y" ]; then args="-v" fi From 58ff4d878557567cbf1e90007192941fc7d83592 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:41:24 +0100 Subject: [PATCH 10/36] shellcheck and shfmt --- global/pre-tasks.d/014set-cosmos-permissions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/pre-tasks.d/014set-cosmos-permissions b/global/pre-tasks.d/014set-cosmos-permissions index e3a802c..7c8d419 100755 --- a/global/pre-tasks.d/014set-cosmos-permissions +++ b/global/pre-tasks.d/014set-cosmos-permissions @@ -16,7 +16,7 @@ if ! test -d "$COSMOS_BASE"; then fi args="" -if [ "x$COSMOS_VERBOSE" = "xy" ]; then +if [ "$COSMOS_VERBOSE" = "y" ]; then args="-v" fi From d81c6fff09514097b7b547e966cd5b79001d099b Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:42:59 +0100 Subject: [PATCH 11/36] shellcheck and shfmt --- global/post-tasks.d/999reboot | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/global/post-tasks.d/999reboot b/global/post-tasks.d/999reboot index 17d37aa..6f48502 100755 --- a/global/post-tasks.d/999reboot +++ b/global/post-tasks.d/999reboot @@ -20,13 +20,13 @@ if [[ -f /var/run/reboot-required && -f /etc/cosmos-automatic-reboot ]]; then sleep=15 elif [[ $HOSTNAME =~ -fre- ]]; then # reboot hosts in site FRE with 15+180 to 15+180+180 seconds delay - sleep=$((180 + ($RANDOM % 180))) + sleep=$((180 + (RANDOM % 180))) elif [[ $HOSTNAME =~ -lla- ]]; then # reboot hosts in site LLA with 15+180+180 to 15+180+180+180 seconds delay - sleep=$((375 + ($RANDOM % 180))) + sleep=$((375 + (RANDOM % 180))) else # reboot hosts in any other site with 15 to 315 seconds delay - sleep=$((15 + ($RANDOM % 300))) + sleep=$((15 + (RANDOM % 300))) fi logger -p local0.emerg -i -t cosmos-automatic-reboot "Rebooting automatically in $sleep seconds (if /var/run/reboot-required still exists)" From 36c7ddd8467a9fa452c51d0ac27df930a6a7a256 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:43:35 +0100 Subject: [PATCH 12/36] shellcheck and shfmt --- global/post-tasks.d/099autoremove | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/post-tasks.d/099autoremove b/global/post-tasks.d/099autoremove index cc46a0f..31e9fc2 100755 --- a/global/post-tasks.d/099autoremove +++ b/global/post-tasks.d/099autoremove @@ -2,7 +2,7 @@ export DEBIAN_FRONTEND='noninteractive' -if (($RANDOM % 20 == 0)); then +if ((RANDOM % 20 == 0)); then apt-get -qq update apt-get -qq -y autoremove fi From acb8124949e3c5dfe521e7cf0a14ae21f7ee43dc Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:44:15 +0100 Subject: [PATCH 13/36] shellcheck and shfmt --- global/post-tasks.d/030puppet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/global/post-tasks.d/030puppet b/global/post-tasks.d/030puppet index 5cc09c9..c0bd418 100755 --- a/global/post-tasks.d/030puppet +++ b/global/post-tasks.d/030puppet @@ -3,7 +3,7 @@ set -e args=() -if [ "x$COSMOS_VERBOSE" = "xy" ]; then +if [ "$COSMOS_VERBOSE" = "y" ]; then args+=('--verbose') args+=('--show_diff') else @@ -12,7 +12,7 @@ fi if [ -f /usr/bin/puppet ] && [ -d /etc/puppet/manifests ]; then find /etc/puppet/manifests -name \*.pp | while read -r m; do - test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m" + test "$COSMOS_VERBOSE" = "y" && echo "$0: Applying Puppet manifest $m" puppet apply "${args[@]}" "$m" done From 3e5a32a213f4721542362ebf7b3e8aa2a7d57f1b Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:56:30 +0100 Subject: [PATCH 14/36] shellcheck and shfmt --- global/post-tasks.d/018packages | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index e223326..4236799 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -18,7 +18,7 @@ stage_module() { git archive --format=tar --prefix="$1"/ "$2" | (cd "$CACHE_DIR/staging/" && tar xf -) } -if [ -f "$CONFIG" -o "$LOCALCONFIG" ]; then +if [ -f "$CONFIG" ] || [ "$LOCALCONFIG" ]; then if [ ! -d "$MODULES_DIR" ]; then mkdir -p "$MODULES_DIR" fi @@ -37,9 +37,9 @@ if [ -f "$CONFIG" -o "$LOCALCONFIG" ]; then # First pass to clone any new modules, and update those marked for updating. grep -h -E -v "^#" "${files[@]}" | sort | ( - while read module src update pattern; do + while read -r module src update pattern; do # We only support git://, file:/// and https:// urls at the moment - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then + if [ "${src:0:6}" = "git://" ] || [ "${src:0:8}" = "file:///" ] || [ "${src:0:8}" = "https://" ]; then if [ ! -d "$CACHE_DIR/scm/$module" ]; then git clone -q "$src" "$CACHE_DIR/scm/$module" elif [ -d "$CACHE_DIR/scm/$module/.git" ]; then @@ -76,9 +76,9 @@ if [ -f "$CONFIG" -o "$LOCALCONFIG" ]; then # Second pass to verify the signatures on all modules and stage those that # have good signatures. grep -h -E -v "^#" "${files[@]}" | sort | ( - while read module src update pattern; do + while read -r module src update pattern; do # We only support git://, file:/// and https:// urls at the moment - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then + if [ "${src:0:6}" = "git://" ] || [ "${src:0:8}" = "file:///" ] || [ "${src:0:8}" = "https://" ]; then # Verify git tag cd "$CACHE_DIR/scm/$module" || exit 1 TAG=$(git tag -l "${pattern:-*}" | sort | tail -1) @@ -89,8 +89,7 @@ if [ -f "$CONFIG" -o "$LOCALCONFIG" ]; then echo -e "${red}ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module ${module}${reset}" continue fi - git tag -v "$TAG" &>/dev/null - if [ $? == 0 ]; then + if git tag -v "$TAG" &>/dev/null; then #if [ "$COSMOS_VERBOSE" = "y" ]; thengg # # short output on good signature # git tag -v $TAG 2>&1 | grep "gpg: Good signature" @@ -107,7 +106,7 @@ if [ -f "$CONFIG" -o "$LOCALCONFIG" ]; then ) # Cleanup removed puppet modules from CACHE_DIR - for MODULE in $(ls -1 $CACHE_DIR/staging/); do + for MODULE in "$CACHE_DIR"/staging/*; do if ! grep -h -E -q "^$MODULE\s+" "$CONFIG" "$LOCALCONFIG"; then rm -rf "$CACHE_DIR"/{scm,staging}/"$MODULE" fi From 634f5ac5e4973df23481e07a647e55a43b0ebc0d Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 15:58:54 +0100 Subject: [PATCH 15/36] shellcheck and shfmt --- global/post-tasks.d/014set-cosmos-permissions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/post-tasks.d/014set-cosmos-permissions b/global/post-tasks.d/014set-cosmos-permissions index e3a802c..7c8d419 100755 --- a/global/post-tasks.d/014set-cosmos-permissions +++ b/global/post-tasks.d/014set-cosmos-permissions @@ -16,7 +16,7 @@ if ! test -d "$COSMOS_BASE"; then fi args="" -if [ "x$COSMOS_VERBOSE" = "xy" ]; then +if [ "$COSMOS_VERBOSE" = "y" ]; then args="-v" fi From 624f9f01886704c16c11ae8b5590c91ed9954200 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 16:11:39 +0100 Subject: [PATCH 16/36] shellcheck and shfmt --- global/overlay/usr/local/sbin/cosmos_vm | 136 ++++++++++++++++-------- 1 file changed, 92 insertions(+), 44 deletions(-) diff --git a/global/overlay/usr/local/sbin/cosmos_vm b/global/overlay/usr/local/sbin/cosmos_vm index 5eec8f7..230fc12 100755 --- a/global/overlay/usr/local/sbin/cosmos_vm +++ b/global/overlay/usr/local/sbin/cosmos_vm @@ -1,9 +1,9 @@ #!/bin/bash -set -- $(getopt h?H:D?s:B:M:C:R:i:g:n:I:G:N: "$@") +set -- "$(getopt h?H:D?s:B:M:C:R:i:g:n:I:G:N: "$@")" src_image="" -size="1G" +size="2G" dhcp="no" hostname="default" bridge="br0" @@ -20,27 +20,75 @@ gateway6="" netmask6="64" while [ $# -gt 0 ]; do - case "$1" in - (-h) echo "Usage: $0 [-h] [-H hostname] [-M ] [-C <#cpus>] [-B ] [-D (dhcp)] [-i/-I ] [-n/-N ] [-g/-G ] [-R ] [-s ]"; exit 0;; - (-H) hostname="$2"; shift;; - (-s) src_image="$2"; shift;; - (-D) dhcp="yes" ;; - (-S) size="$2"; shift;; - (-B) bridge="$2"; shift;; - (-M) mem="$2"; shift;; - (-C) cpus="$2"; shift;; - (-R) resolver="$2"; shift;; - (-i) ip="$2"; shift;; - (-g) gateway="$2"; shift;; - (-n) netmask="$2"; shift;; - (-I) ip6="$2"; shift;; - (-G) gateway6="$2"; shift;; - (-N) netmask6="$2"; shift;; - (--) shift; break;; - (-*) echo "Unknown option $1\nUsage: $0 [-h] [-H hostname] [-M ] [-C <#cpus>] [-B ] [-D (dhcp)] [-i/-I ] [-n/-N ] [-g/-G ] [-R ] [-s ]"; exit 1;; - (*) break;; - esac - shift + case "$1" in + -h) + echo "Usage: $0 [-h] [-H hostname] [-M ] [-C <#cpus>] [-B ] [-D (dhcp)] [-i/-I ] [-n/-N ] [-g/-G ] [-R ] [-s ]" + exit 0 + ;; + -H) + hostname="$2" + shift + ;; + -s) + src_image="$2" + shift + ;; + -D) dhcp="yes" ;; + -S) + size="$2" + shift + ;; + -B) + bridge="$2" + shift + ;; + -M) + mem="$2" + shift + ;; + -C) + cpus="$2" + shift + ;; + -R) + resolver="$2" + shift + ;; + -i) + ip="$2" + shift + ;; + -g) + gateway="$2" + shift + ;; + -n) + netmask="$2" + shift + ;; + -I) + ip6="$2" + shift + ;; + -G) + gateway6="$2" + shift + ;; + -N) + netmask6="$2" + shift + ;; + --) + shift + break + ;; + -*) + printf "Unknown option %s\nUsage: %s [-h] [-H hostname] [-M ] [-C <#cpus>] [-B ] [-D (dhcp)] [-i/-I ] [-n/-N ] [-g/-G ] [-R ] [-s ]" "$1" "$0" + exit 1 + ;; + *) break ;; + esac + shift done id=$(uuidgen) @@ -48,12 +96,12 @@ id=$(uuidgen) seed=${id}_seed.img disk=${id}.img -rm -f ${seed} -truncate --size 2M ${seed} -mkfs.vfat -n cidata ${seed} 2>/dev/null +rm -f "${seed}" +truncate --size "${size}" "${seed}" +mkfs.vfat -n cidata "${seed}" 2>/dev/null user_data=$(mktemp) -cat > ${user_data} <"${user_data}" < ${meta_data} <"${meta_data}" <> ${meta_data} <>"${meta_data}" <> ${meta_data} <>"${meta_data}" <> ${meta_data} <>"${meta_data}" </dev/null -mcopy -i ${seed} ${meta_data} ::meta-data 2>/dev/null -mcopy -i ${seed} /etc/cosmos/apt/bootstrap-cosmos.sh /etc/cosmos/apt/cosmos_1.5-1_all.deb :: -mv ${seed} /var/lib/libvirt/images/ +mcopy -i "${seed}" "${user_data}" ::user-data 2>/dev/null +mcopy -i "${seed}" "${meta_data}" ::meta-data 2>/dev/null +mcopy -i "${seed}" /etc/cosmos/apt/bootstrap-cosmos.sh /etc/cosmos/apt/cosmos_1.5-1_all.deb :: +mv "${seed}" /var/lib/libvirt/images/ virsh pool-refresh default -virsh vol-clone --pool default ${src_image} ${disk} +virsh vol-clone --pool default "${src_image}" "${disk}" -virt-install -r ${mem} -n ${hostname} --vcpus=${cpus} --autostart --memballoon virtio --network bridge=${bridge} --boot hd --disk vol=default/${disk},format=qcow2,bus=virtio --disk vol=default/${seed},bus=virtio +virt-install -r "${mem}" -n "${hostname}" --vcpus="${cpus}" --autostart --memballoon virtio --network bridge="${bridge}" --boot hd --disk vol=default/"${disk}",format=qcow2,bus=virtio --disk vol=default/"${seed}",bus=virtio -rm -f ${user_data} -rm -f ${meta_data} +rm -f "${user_data}" +rm -f "${meta_data}" From 63a13a8a7fb68183ec092b13cb208807afcb88b1 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 16:15:17 +0100 Subject: [PATCH 17/36] shellcheck and shfmt --- .../overlay/usr/local/libexec/cosmos-cron-wrapper | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/global/overlay/usr/local/libexec/cosmos-cron-wrapper b/global/overlay/usr/local/libexec/cosmos-cron-wrapper index 4c7bf06..1c42a6f 100755 --- a/global/overlay/usr/local/libexec/cosmos-cron-wrapper +++ b/global/overlay/usr/local/libexec/cosmos-cron-wrapper @@ -2,11 +2,17 @@ test -f /etc/no-automatic-cosmos && exit 0 -RUN_COSMOS='/usr/local/bin/run-cosmos --random-sleep' -SCRIPTHERDER_CMD='' +RUN_COSMOS=('/usr/local/bin/run-cosmos' '--random-sleep') +SCRIPTHERDER_CMD=() if [ -x /usr/local/bin/scriptherder ]; then - SCRIPTHERDER_CMD='/usr/local/bin/scriptherder --mode wrap --syslog --name cosmos --' + SCRIPTHERDER_CMD+=('/usr/local/bin/scriptherder') + SCRIPTHERDER_CMD+=('--mode') + SCRIPTHERDER_CMD+=('wrap') + SCRIPTHERDER_CMD+=('--syslog') + SCRIPTHERDER_CMD+=('--name') + SCRIPTHERDER_CMD+=('cosmos') + SCRIPTHERDER_CMD+=('--') fi -exec ${SCRIPTHERDER_CMD} ${RUN_COSMOS} "$@" +exec "${SCRIPTHERDER_CMD[@]}" "${RUN_COSMOS[@]}" "$@" From 0f80724b25e3883bd20fe8da6c4e12bd1f86305a Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 25 Nov 2025 16:17:20 +0100 Subject: [PATCH 18/36] shellcheck and shfmt --- global/overlay/usr/local/bin/run-cosmos | 223 ++++++++++++------------ 1 file changed, 112 insertions(+), 111 deletions(-) diff --git a/global/overlay/usr/local/bin/run-cosmos b/global/overlay/usr/local/bin/run-cosmos index f3e2cb0..f868beb 100755 --- a/global/overlay/usr/local/bin/run-cosmos +++ b/global/overlay/usr/local/bin/run-cosmos @@ -5,7 +5,8 @@ set -e -readonly PROGNAME=$(basename "$0") +PROGNAME="$(basename "$0")" +readonly PROGNAME readonly LOCKFILE_DIR=/tmp readonly LOCK_FD=200 readonly FLEETLOCK_CONFIG=/etc/run-cosmos-fleetlock-conf @@ -18,150 +19,150 @@ readonly HEALTHCHECK_DISABLE_FILE=/etc/run-cosmos-healthcheck-disable export GIT_PAGER=cat lock() { - local prefix=$1 - local fd=${2:-$LOCK_FD} - local lock_file=$LOCKFILE_DIR/$prefix.lock + local prefix=$1 + local fd=${2:-$LOCK_FD} + local lock_file=$LOCKFILE_DIR/$prefix.lock - # create lock file - eval "exec $fd>$lock_file" + # create lock file + eval "exec $fd>$lock_file" - # acquier the lock - flock -n "$fd" \ - && return 0 \ - || return 1 + # acquier the lock + flock -n "$fd" && + return 0 || + return 1 } eexit() { - local error_str="$*" + local error_str="$*" - echo "$error_str" - exit 1 + echo "$error_str" + exit 1 } oexit() { - local info_str="$*" + local info_str="$*" - echo "$info_str" - exit 0 + echo "$info_str" + exit 0 } fleetlock_enable_unlock_service() { - # In case e.g. the unit file has been removed "FragmentPath" will still - # return the old filename until daemon-reload is called, so do that here - # before we try checking for the FragmentPath. - need_reload=$(systemctl show --property NeedDaemonReload $FLEETLOCK_UNLOCK_SERVICE | awk -F= '{print $2}') - if [ "$need_reload" = "yes" ]; then - systemctl daemon-reload - fi - - unit_file=$(systemctl show --property FragmentPath $FLEETLOCK_UNLOCK_SERVICE | awk -F= '{print $2}') - if [ -z "$unit_file" ]; then - # No unit file matching the service name, do nothing - return 0 - fi - - # Enable the service if needed - systemctl is-enabled --quiet $FLEETLOCK_UNLOCK_SERVICE || systemctl enable --quiet $FLEETLOCK_UNLOCK_SERVICE + # In case e.g. the unit file has been removed "FragmentPath" will still + # return the old filename until daemon-reload is called, so do that here + # before we try checking for the FragmentPath. + need_reload=$(systemctl show --property NeedDaemonReload $FLEETLOCK_UNLOCK_SERVICE | awk -F= '{print $2}') + if [ "$need_reload" = "yes" ]; then + systemctl daemon-reload + fi + + unit_file=$(systemctl show --property FragmentPath $FLEETLOCK_UNLOCK_SERVICE | awk -F= '{print $2}') + if [ -z "$unit_file" ]; then + # No unit file matching the service name, do nothing + return 0 + fi + + # Enable the service if needed + systemctl is-enabled --quiet $FLEETLOCK_UNLOCK_SERVICE || systemctl enable --quiet $FLEETLOCK_UNLOCK_SERVICE } fleetlock_lock() { - if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then - # Make sure the unlock service is enabled before we take a lock if - # cosmos ends up rebooting the machine before fleetlock_unlock() is - # called. - fleetlock_enable_unlock_service || return 1 - local fleetlock_group="" - local optional_args=() - # shellcheck source=/dev/null - . $FLEETLOCK_CONFIG || return 1 - if [ -z "$fleetlock_group" ]; then - echo "Unable to set fleetlock_group" - return 1 - fi - if [ -n "$fleetlock_lock_timeout" ]; then - optional_args+=("--timeout") - optional_args+=("$fleetlock_lock_timeout") - fi - echo "Getting fleetlock lock" - $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --lock "${optional_args[@]}" || return 1 - fi - return 0 + if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then + # Make sure the unlock service is enabled before we take a lock if + # cosmos ends up rebooting the machine before fleetlock_unlock() is + # called. + fleetlock_enable_unlock_service || return 1 + local fleetlock_group="" + local optional_args=() + # shellcheck source=/dev/null + . $FLEETLOCK_CONFIG || return 1 + if [ -z "$fleetlock_group" ]; then + echo "Unable to set fleetlock_group" + return 1 + fi + if [ -n "$fleetlock_lock_timeout" ]; then + optional_args+=("--timeout") + optional_args+=("$fleetlock_lock_timeout") + fi + echo "Getting fleetlock lock" + $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --lock "${optional_args[@]}" || return 1 + fi + return 0 } fleetlock_unlock() { - if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then - local fleetlock_group="" - local optional_args=() - # shellcheck source=/dev/null - . $FLEETLOCK_CONFIG || return 1 - if [ -z "$fleetlock_group" ]; then - echo "Unable to set fleetlock_group" - return 1 - fi - if [ -n "$fleetlock_unlock_timeout" ]; then - optional_args+=("--timeout") - optional_args+=("$fleetlock_unlock_timeout") - fi - machine_is_healthy || return 1 - echo "Releasing fleetlock lock" - $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --unlock "${optional_args[@]}" || return 1 - fi - return 0 + if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then + local fleetlock_group="" + local optional_args=() + # shellcheck source=/dev/null + . $FLEETLOCK_CONFIG || return 1 + if [ -z "$fleetlock_group" ]; then + echo "Unable to set fleetlock_group" + return 1 + fi + if [ -n "$fleetlock_unlock_timeout" ]; then + optional_args+=("--timeout") + optional_args+=("$fleetlock_unlock_timeout") + fi + machine_is_healthy || return 1 + echo "Releasing fleetlock lock" + $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --unlock "${optional_args[@]}" || return 1 + fi + return 0 } machine_is_healthy() { - if [ ! -f $HEALTHCHECK_DISABLE_FILE ] && [ -x $HEALTHCHECK_TOOL ]; then - local fleetlock_healthcheck_timeout="" - local optional_args=() - # shellcheck source=/dev/null - . $FLEETLOCK_CONFIG || return 1 - if [ -n "$fleetlock_healthcheck_timeout" ]; then - optional_args+=("--timeout") - optional_args+=("$fleetlock_healthcheck_timeout") - fi - echo "Running any health checks" - $HEALTHCHECK_TOOL "${optional_args[@]}" || return 1 - fi - return 0 + if [ ! -f $HEALTHCHECK_DISABLE_FILE ] && [ -x $HEALTHCHECK_TOOL ]; then + local fleetlock_healthcheck_timeout="" + local optional_args=() + # shellcheck source=/dev/null + . $FLEETLOCK_CONFIG || return 1 + if [ -n "$fleetlock_healthcheck_timeout" ]; then + optional_args+=("--timeout") + optional_args+=("$fleetlock_healthcheck_timeout") + fi + echo "Running any health checks" + $HEALTHCHECK_TOOL "${optional_args[@]}" || return 1 + fi + return 0 } -main () { - if [[ $1 == '--random-sleep' ]]; then - shift - sleep=$((RANDOM % 300)) +main() { + if [[ $1 == '--random-sleep' ]]; then + shift + sleep=$((RANDOM % 300)) - echo "$0: Sleeping for ${sleep} seconds before attempting to run cosmos" - sleep $sleep - fi + echo "$0: Sleeping for ${sleep} seconds before attempting to run cosmos" + sleep $sleep + fi - lock "$PROGNAME" || eexit "Only one instance of $PROGNAME can run at one time." - fleetlock_lock || eexit "Unable to acquire fleetlock lock." - cosmos "$@" update - cosmos "$@" apply + lock "$PROGNAME" || eexit "Only one instance of $PROGNAME can run at one time." + fleetlock_lock || eexit "Unable to acquire fleetlock lock." + cosmos "$@" update + cosmos "$@" apply - if [ -f /var/run/reboot-required ] && [ -f /var/run/cosmos-reboot-in-progress ]; then - oexit "${PROGNAME}: Will not attempt fleetlock_unlock (exiting early) due to existing reboot files" - fi + if [ -f /var/run/reboot-required ] && [ -f /var/run/cosmos-reboot-in-progress ]; then + oexit "${PROGNAME}: Will not attempt fleetlock_unlock (exiting early) due to existing reboot files" + fi - fleetlock_unlock || eexit "Unable to release fleetlock lock." + fleetlock_unlock || eexit "Unable to release fleetlock lock." - touch /var/run/last-cosmos-ok.stamp + touch /var/run/last-cosmos-ok.stamp - if [ -f /cosmos-reboot ]; then - rm -f /cosmos-reboot - reboot - fi + if [ -f /cosmos-reboot ]; then + rm -f /cosmos-reboot + reboot + fi } # Most of the time we just pass on any arguments to the underlying cosmos # tools, if adding special cases here make sure to not shadow any arguments # (like "-v") which users expect to be passed on to cosmos. case "$1" in - "fleetlock-unlock") - lock "$PROGNAME" || oexit "$PROGNAME appears locked by a running run-cosmos, let it handle unlocking instead." - fleetlock_unlock || eexit "Unable to release fleetlock lock." - ;; - *) - main "$@" - ;; +"fleetlock-unlock") + lock "$PROGNAME" || oexit "$PROGNAME appears locked by a running run-cosmos, let it handle unlocking instead." + fleetlock_unlock || eexit "Unable to release fleetlock lock." + ;; +*) + main "$@" + ;; esac From f84e6808166b37a2a01f6b140cd89d08b705917e Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Nov 2025 12:50:00 +0100 Subject: [PATCH 19/36] shellcheck and shfmt --- .../etc/cosmos/apt/bootstrap-cosmos.sh | 82 +++++++++++-------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index 5e27f3d..f87be47 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -1,28 +1,27 @@ -#!/bin/sh +#!/bin/bash set -e cmd_hostname="$1" if test -z "$cmd_hostname"; then - echo "Usage: $0 HOSTNAME REPO TAGPATTERN" - exit 1 + echo "Usage: $0 HOSTNAME REPO TAGPATTERN" + exit 1 fi cmd_repo="$2" if test -z "$cmd_repo"; then - echo "Usage $0 HOSTNAME REPO TAGPATTERN" - exit 2 + echo "Usage $0 HOSTNAME REPO TAGPATTERN" + exit 2 fi cmd_tags="$3" if test -z "$cmd_tags"; then - echo "Usage $0 HOSTNAME REPO TAGPATTERN" - exit 3 + echo "Usage $0 HOSTNAME REPO TAGPATTERN" + exit 3 fi set -x - # cloud-init runs with LANG='US-ASCII' which is likely to fail because of non-US-ASCII chars in the manifest export LANG='en_US.UTF-8' @@ -31,26 +30,27 @@ export DEBIAN_FRONTEND='noninteractive' apt-get -y update apt-get -y upgrade for pkg in rsync git git-core wget gpg jq; do - # script is running with "set -e", use "|| true" to allow packages to not - # exist without stopping the script - apt-get -y install $pkg || true + # script is running with "set -e", use "|| true" to allow packages to not + # exist without stopping the script + apt-get -y install $pkg || true done cosmos_deb=$(find ./ -maxdepth 1 -name 'cosmos_*.deb' | sort -V | tail -1) dpkg -i "$cosmos_deb" if ! test -d /var/cache/cosmos/repo; then - cosmos clone "$cmd_repo" + cosmos clone "$cmd_repo" fi # Re-run cosmos at reboot until it succeeds - use bash -l to get working proxy settings. # It is possible the file does not exist or contains no matching lines, # both cases are OK -grep -v "^exit 0" /etc/rc.local > /etc/rc.local.new || true -(echo "" - echo "test -f /etc/run-cosmos-at-boot && (bash -l cosmos -v update; bash -l cosmos -v apply && rm /etc/run-cosmos-at-boot)" - echo "" - echo "exit 0" -) >> /etc/rc.local.new +grep -v "^exit 0" /etc/rc.local >/etc/rc.local.new || true +( + echo "" + echo "test -f /etc/run-cosmos-at-boot && (bash -l cosmos -v update; bash -l cosmos -v apply && rm /etc/run-cosmos-at-boot)" + echo "" + echo "exit 0" +) >>/etc/rc.local.new mv -f /etc/rc.local.new /etc/rc.local touch /etc/run-cosmos-at-boot @@ -58,9 +58,9 @@ touch /etc/run-cosmos-at-boot # If this cloud-config is set, it will interfere with our changes to /etc/hosts # The configuration seems to move around between cloud-config versions for file in /etc/cloud/cloud.cfg /etc/cloud/cloud.cfg.d/01_debian_cloud.cfg; do - if [ -f ${file} ]; then - sed -i 's/manage_etc_hosts: true/manage_etc_hosts: false/g' ${file} - fi + if [ -f ${file} ]; then + sed -i 's/manage_etc_hosts: true/manage_etc_hosts: false/g' ${file} + fi done # Remove potential $hostname.novalocal, added by cloud-init or Debian default @@ -76,32 +76,41 @@ version=$(lsb_release -rs) min_version=1337 host_ip=127.0.1.1 if [ "${vendor}" = "Ubuntu" ]; then - min_version=20.04 + min_version=20.04 elif [ "${vendor}" = "Debian" ]; then - min_version=11 + min_version=11 fi -hostname $cmd_hostname -short=`echo ${cmd_hostname} | awk -F. '{print $1}'` +hostname "$cmd_hostname" +short=$(echo "${cmd_hostname}" | awk -F. '{print $1}') # Only change behavior on modern OS where `ip -j` outputs a json predictuble # enought to work with. # # Use `dpkg` to easier compare ubuntu versions. if dpkg --compare-versions "${version}" "ge" "${min_version}"; then - # When hostname pointed to loopback in /etc/hosts containers running on the - # host tried to connect to the container itself instead of the host. - host_ip=$(ip -j address show "$(ip -j route show default | jq -r '.[0].dev')" | jq -r .[0].addr_info[0].local) + # When hostname pointed to loopback in /etc/hosts containers running on the + # host tried to connect to the container itself instead of the host. + host_ip=$(ip -j address show "$(ip -j route show default | jq -r '.[0].dev')" | jq -r .[0].addr_info[0].local) fi -echo "${host_ip} ${cmd_hostname} ${short}" >> /etc/hosts +echo "${host_ip} ${cmd_hostname} ${short}" >>/etc/hosts # Set up cosmos models. They are in the order of most significant first, so we want # -_host_type=`echo $cmd_hostname | cut -d - -f 1` +_host_type=$(echo "$cmd_hostname" | cut -d - -f 1) +# $COSMOS_REPO should be handles as $PATH without expansion at this stage +# shellcheck disable=SC2016 +models_array=('$COSMOS_REPO/'"$cmd_hostname/") +if [ -d "/var/cache/cosmos/repo/${_host_type}-common" ]; then + # shellcheck disable=SC2016 + models_array+=('$COSMOS_REPO/'"${_host_type}-common/") +fi +# shellcheck disable=SC2016 +models_array+=('$COSMOS_REPO/global/') models=$( - echo -n '\\$COSMOS_REPO/'"$cmd_hostname/:" - test -d /var/cache/cosmos/repo/${_host_type}-common && echo -n '\\$COSMOS_REPO/'"${_host_type}-common/:" - echo -n '\\$COSMOS_REPO/global/' + IFS=: + echo "${models_array[*]}" ) + echo "Configuring cosmos with the following models:" echo "${models}" @@ -112,7 +121,10 @@ env COSMOS_BASE=/var/cache/cosmos COSMOS_KEYS=/var/cache/cosmos/repo/global/over mkdir -p /var/cache/scriptherder -(date; nohup cosmos -v update && nohup cosmos -v apply && rm /etc/run-cosmos-at-boot; date) 2>&1 | tee /var/log/cosmos.log - +( + date + nohup cosmos -v update && nohup cosmos -v apply && rm /etc/run-cosmos-at-boot + date +) 2>&1 | tee /var/log/cosmos.log exit 0 From 2acc17d2b200386c58f92cbfa49d76fe35bb3eaf Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Nov 2025 13:18:49 +0100 Subject: [PATCH 20/36] Automatic shellcheck? --- .github/workflows/reviewdog.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/reviewdog.yml diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml new file mode 100644 index 0000000..c46adb4 --- /dev/null +++ b/.github/workflows/reviewdog.yml @@ -0,0 +1,16 @@ +name: reviewdog +on: + push: + pull_request: + branches: [ "master" ] +jobs: + shellcheck: + name: runner / shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: shellcheck + uses: reviewdog/action-shellcheck@v1.32.0 + with: + reporter: github-pr-review + check_all_files_with_shebangs: "true" From 4b82be4c51fb4b86250fb3da46ab4e33c59abba5 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Nov 2025 13:23:32 +0100 Subject: [PATCH 21/36] Default branch is main --- .github/workflows/reviewdog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index c46adb4..e4a77d5 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -2,7 +2,7 @@ name: reviewdog on: push: pull_request: - branches: [ "master" ] + branches: [ "main" ] jobs: shellcheck: name: runner / shellcheck From 8f7efc6876b79c330fb63916d4a4aaa93d19c70a Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Nov 2025 13:26:26 +0100 Subject: [PATCH 22/36] Suggestion from Sonar --- .github/workflows/reviewdog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index e4a77d5..b1a5a56 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: shellcheck - uses: reviewdog/action-shellcheck@v1.32.0 + uses: reviewdog/action-shellcheck@4c07458293ac342d477251099501a718ae5ef86e # v1.32.0 with: reporter: github-pr-review check_all_files_with_shebangs: "true" From 47ff3566e9e553f15c166a503552e074a1287f9c Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Nov 2025 13:40:32 +0100 Subject: [PATCH 23/36] Run shfmt aswell --- .github/workflows/{reviewdog.yml => shellcheck.yml} | 2 +- .github/workflows/shfmt.yml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) rename .github/workflows/{reviewdog.yml => shellcheck.yml} (95%) create mode 100644 .github/workflows/shfmt.yml diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/shellcheck.yml similarity index 95% rename from .github/workflows/reviewdog.yml rename to .github/workflows/shellcheck.yml index b1a5a56..f1950c3 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/shellcheck.yml @@ -1,4 +1,4 @@ -name: reviewdog +name: shellcheck on: push: pull_request: diff --git a/.github/workflows/shfmt.yml b/.github/workflows/shfmt.yml new file mode 100644 index 0000000..65f2c15 --- /dev/null +++ b/.github/workflows/shfmt.yml @@ -0,0 +1,9 @@ +name: shfmt +on: [pull_request] +jobs: + shfmt: + name: runner / shfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: reviewdog/action-shfmt@d8f080930b9be5847b4f97e9f4122b81a82aaeac # v1.0.4 From ee20d903d09ef0e929f476ead43aa77e758599d8 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Nov 2025 14:23:28 +0100 Subject: [PATCH 24/36] Variable reporter --- .github/workflows/shellcheck.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index f1950c3..d9499a9 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -9,8 +9,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + - uses: haya14busa/action-cond@94f77f7a80cd666cb3155084e428254fea4281fd # v1.2.1 + id: reporter + with: + cond: ${{ github.event_name == 'pull_request' }} + if_true: "github-pr-review" + if_false: "github-check" - name: shellcheck uses: reviewdog/action-shellcheck@4c07458293ac342d477251099501a718ae5ef86e # v1.32.0 with: - reporter: github-pr-review + reporter: ${{ steps.reporter.outputs.value }} check_all_files_with_shebangs: "true" From 20e36b646b32b8c36e59b0391b780b049989a6dc Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Nov 2025 14:29:13 +0100 Subject: [PATCH 25/36] Test if shellcheck runs --- bump-tag | 2 +- host-puppet-conf-test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bump-tag b/bump-tag index 9352245..7741401 100755 --- a/bump-tag +++ b/bump-tag @@ -33,7 +33,7 @@ fi if [[ -n "${tag}" ]]; then tagpfx="${tag}" else - tagpfx="${deftag}" + tagpfx=${deftag} fi # This is the current branch that Git will diff against. diff --git a/host-puppet-conf-test b/host-puppet-conf-test index 753dc8c..78f03d2 100755 --- a/host-puppet-conf-test +++ b/host-puppet-conf-test @@ -17,7 +17,7 @@ fi # Check if cosmos or puppet is already running on host echo "Checking if puppet or cosmos is already running..." -ssh root@"$HOSTNAME" ps aux | grep -Ev "grep|edit-secrets|gpg-agent" | grep -Eq "cosmos|puppet" +ssh root@$HOSTNAME ps aux | grep -Ev "grep|edit-secrets|gpg-agent" | grep -Eq "cosmos|puppet" if [ $? -eq 1 ]; then echo "Copying files to host..." From d75a04956c24214af6038c2226e6889b7af06d8e Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Nov 2025 14:31:18 +0100 Subject: [PATCH 26/36] Revert "Test if shellcheck runs" This reverts commit 20e36b646b32b8c36e59b0391b780b049989a6dc. --- bump-tag | 2 +- host-puppet-conf-test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bump-tag b/bump-tag index 7741401..9352245 100755 --- a/bump-tag +++ b/bump-tag @@ -33,7 +33,7 @@ fi if [[ -n "${tag}" ]]; then tagpfx="${tag}" else - tagpfx=${deftag} + tagpfx="${deftag}" fi # This is the current branch that Git will diff against. diff --git a/host-puppet-conf-test b/host-puppet-conf-test index 78f03d2..753dc8c 100755 --- a/host-puppet-conf-test +++ b/host-puppet-conf-test @@ -17,7 +17,7 @@ fi # Check if cosmos or puppet is already running on host echo "Checking if puppet or cosmos is already running..." -ssh root@$HOSTNAME ps aux | grep -Ev "grep|edit-secrets|gpg-agent" | grep -Eq "cosmos|puppet" +ssh root@"$HOSTNAME" ps aux | grep -Ev "grep|edit-secrets|gpg-agent" | grep -Eq "cosmos|puppet" if [ $? -eq 1 ]; then echo "Copying files to host..." From e95552436070b89a7fded03fcaed2bdd6934d889 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 27 Nov 2025 08:22:45 +0100 Subject: [PATCH 27/36] Try to override the default --- .github/workflows/shfmt.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/shfmt.yml b/.github/workflows/shfmt.yml index 65f2c15..665cd39 100644 --- a/.github/workflows/shfmt.yml +++ b/.github/workflows/shfmt.yml @@ -7,3 +7,5 @@ jobs: steps: - uses: actions/checkout@v6 - uses: reviewdog/action-shfmt@d8f080930b9be5847b4f97e9f4122b81a82aaeac # v1.0.4 + with: + shfmt_flags: "" From 35d6d278d75949ad75e75ad42c805d9dd244e5b4 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 27 Nov 2025 08:25:14 +0100 Subject: [PATCH 28/36] Run on PRs only --- .github/workflows/shellcheck.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index d9499a9..fdee1e4 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -1,6 +1,5 @@ name: shellcheck on: - push: pull_request: branches: [ "main" ] jobs: From 5531e9c5eb9e930a9aae14c11755a432541a7260 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 27 Nov 2025 12:40:51 +0100 Subject: [PATCH 29/36] Local config is not always present --- global/post-tasks.d/018packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index 4236799..d4d7ca0 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -107,7 +107,7 @@ if [ -f "$CONFIG" ] || [ "$LOCALCONFIG" ]; then # Cleanup removed puppet modules from CACHE_DIR for MODULE in "$CACHE_DIR"/staging/*; do - if ! grep -h -E -q "^$MODULE\s+" "$CONFIG" "$LOCALCONFIG"; then + if ! grep -h -E -q "^$MODULE\s+" "${files[@]}"; then rm -rf "$CACHE_DIR"/{scm,staging}/"$MODULE" fi done From 0e56a7c6635130afc681f11bb183df5090dd62f1 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 28 Nov 2025 11:15:16 +0100 Subject: [PATCH 30/36] Don't expand variable later on Without escape the shell will expand it when perl does it replacement. --- global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index f87be47..1ffc1dc 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -99,13 +99,13 @@ echo "${host_ip} ${cmd_hostname} ${short}" >>/etc/hosts _host_type=$(echo "$cmd_hostname" | cut -d - -f 1) # $COSMOS_REPO should be handles as $PATH without expansion at this stage # shellcheck disable=SC2016 -models_array=('$COSMOS_REPO/'"$cmd_hostname/") +models_array=('\$COSMOS_REPO/'"$cmd_hostname/") if [ -d "/var/cache/cosmos/repo/${_host_type}-common" ]; then # shellcheck disable=SC2016 - models_array+=('$COSMOS_REPO/'"${_host_type}-common/") + models_array+=('\$COSMOS_REPO/'"${_host_type}-common/") fi # shellcheck disable=SC2016 -models_array+=('$COSMOS_REPO/global/') +models_array+=('\$COSMOS_REPO/global/') models=$( IFS=: echo "${models_array[*]}" From eea8839a19c4642cf8241c3642ed909b78537853 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 28 Nov 2025 12:47:41 +0100 Subject: [PATCH 31/36] Cosmos is already run at boot via crontab Also this setup havn't worked for some time (years? dists?): ``` [ 3.082391] systemd-rc-local-generator[269]: /etc/rc.local is not marked executable, skipping. ``` --- global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index 1ffc1dc..16fd40f 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -41,20 +41,6 @@ if ! test -d /var/cache/cosmos/repo; then cosmos clone "$cmd_repo" fi -# Re-run cosmos at reboot until it succeeds - use bash -l to get working proxy settings. -# It is possible the file does not exist or contains no matching lines, -# both cases are OK -grep -v "^exit 0" /etc/rc.local >/etc/rc.local.new || true -( - echo "" - echo "test -f /etc/run-cosmos-at-boot && (bash -l cosmos -v update; bash -l cosmos -v apply && rm /etc/run-cosmos-at-boot)" - echo "" - echo "exit 0" -) >>/etc/rc.local.new -mv -f /etc/rc.local.new /etc/rc.local - -touch /etc/run-cosmos-at-boot - # If this cloud-config is set, it will interfere with our changes to /etc/hosts # The configuration seems to move around between cloud-config versions for file in /etc/cloud/cloud.cfg /etc/cloud/cloud.cfg.d/01_debian_cloud.cfg; do From 83a338d67493a8b723504c160a9b2fc2aaae110b Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 28 Nov 2025 12:50:36 +0100 Subject: [PATCH 32/36] [[ is considered safer And since we now use arrays in bash all the POSIX compatility is under the bus. --- global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index 16fd40f..ba1934d 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -3,19 +3,19 @@ set -e cmd_hostname="$1" -if test -z "$cmd_hostname"; then +if [[ -z "$cmd_hostname" ]]; then echo "Usage: $0 HOSTNAME REPO TAGPATTERN" exit 1 fi cmd_repo="$2" -if test -z "$cmd_repo"; then +if [[ -z "$cmd_repo" ]]; then echo "Usage $0 HOSTNAME REPO TAGPATTERN" exit 2 fi cmd_tags="$3" -if test -z "$cmd_tags"; then +if [[ -z "$cmd_tags" ]]; then echo "Usage $0 HOSTNAME REPO TAGPATTERN" exit 3 fi @@ -37,7 +37,7 @@ done cosmos_deb=$(find ./ -maxdepth 1 -name 'cosmos_*.deb' | sort -V | tail -1) dpkg -i "$cosmos_deb" -if ! test -d /var/cache/cosmos/repo; then +if [[ ! -d /var/cache/cosmos/repo ]]; then cosmos clone "$cmd_repo" fi From d7b3488c12c039d18fb9a9272e38b99b3e846ec7 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 28 Nov 2025 12:53:46 +0100 Subject: [PATCH 33/36] Even more unsafe tests --- global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index ba1934d..1e7cd43 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -61,9 +61,9 @@ vendor=$(lsb_release -is) version=$(lsb_release -rs) min_version=1337 host_ip=127.0.1.1 -if [ "${vendor}" = "Ubuntu" ]; then +if [[ "${vendor}" = "Ubuntu" ]]; then min_version=20.04 -elif [ "${vendor}" = "Debian" ]; then +elif [[] "${vendor}" = "Debian" ]]; then min_version=11 fi @@ -86,7 +86,7 @@ _host_type=$(echo "$cmd_hostname" | cut -d - -f 1) # $COSMOS_REPO should be handles as $PATH without expansion at this stage # shellcheck disable=SC2016 models_array=('\$COSMOS_REPO/'"$cmd_hostname/") -if [ -d "/var/cache/cosmos/repo/${_host_type}-common" ]; then +if [[ -d "/var/cache/cosmos/repo/${_host_type}-common" ]]; then # shellcheck disable=SC2016 models_array+=('\$COSMOS_REPO/'"${_host_type}-common/") fi From bf6865f4e8669914b4e8a7acbe6a2e3cba78680f Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 28 Nov 2025 12:56:58 +0100 Subject: [PATCH 34/36] Syntax error Damn you smart editor --- global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index 1e7cd43..e473c98 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -63,7 +63,7 @@ min_version=1337 host_ip=127.0.1.1 if [[ "${vendor}" = "Ubuntu" ]]; then min_version=20.04 -elif [[] "${vendor}" = "Debian" ]]; then +elif [[ "${vendor}" = "Debian" ]]; then min_version=11 fi From d2d13a7b0af5da7a02bca962e93efa3364389747 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 28 Nov 2025 12:58:12 +0100 Subject: [PATCH 35/36] Missed another unsafe test --- global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index e473c98..675964f 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -44,7 +44,7 @@ fi # If this cloud-config is set, it will interfere with our changes to /etc/hosts # The configuration seems to move around between cloud-config versions for file in /etc/cloud/cloud.cfg /etc/cloud/cloud.cfg.d/01_debian_cloud.cfg; do - if [ -f ${file} ]; then + if [[ -f ${file} ]]; then sed -i 's/manage_etc_hosts: true/manage_etc_hosts: false/g' ${file} fi done From 485f1fb4b92a7c2209cc36574f790a828a22f11b Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 28 Nov 2025 13:39:55 +0100 Subject: [PATCH 36/36] File is no longer created --- global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index 675964f..220dc89 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -109,7 +109,7 @@ mkdir -p /var/cache/scriptherder ( date - nohup cosmos -v update && nohup cosmos -v apply && rm /etc/run-cosmos-at-boot + nohup cosmos -v update && nohup cosmos -v apply date ) 2>&1 | tee /var/log/cosmos.log