commit 98dce030f8c2eb7555d95da9b644375cc48ec1e2 Author: Starbeamrainbowlabs Date: Thu Nov 25 23:26:17 2021 +0000 Import from https://git.starbeamrainbowlabs.com/sbrl/cluster-config/ diff --git a/execute b/execute new file mode 100755 index 0000000..d6f257c --- /dev/null +++ b/execute @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -e; + +# Interesting chars: +# 🛰 - Satellite - looks very cool indeed in Firefox + +lantern_path="../lantern-build-engine"; +if [[ -z "${do_parallel}" ]]; then + do_parallel="true"; +fi + +############################################################################### + +# Make sure the current directory is the location of this script to simplify matters +cd "$(dirname "$(readlink -f "$0")")"; + +# Check out the lantern git submodule if needed +if [ ! -f "${lantern_path}/lantern.sh" ]; then git submodule update --init "${lantern_path}"; fi + +#shellcheck disable=SC1090 +source "${lantern_path}/lantern.sh"; + +############################################################################### + +if [[ -z "${2}" ]]; then + echo "Usage:"; + echo " ./execute {{path/to/jobfile}} {{path/to/hostsfile_a}} {{path/to/hostsfile_n}}"; + exit 1; +fi + +job_filepath="$1"; +if [[ ! -f "${job_filepath}" ]]; then + job_filepath="jobs/${job_filepath}/${job_filepath}.job"; +fi + +shift; + +hosts=""; +while [[ ! -z "${1}" ]]; do + hosts_filename="${1}"; + if [[ ! -f "${hosts_filename}" ]]; then + hosts_filename="hosts/${hosts_filename}.txt"; + fi + + hosts="$(echo "${hosts}" && cat "${hosts_filename}")"; + shift; +done + +ssh_configfile="./ssh-config"; + +if [[ ! -x "${job_filepath}" ]]; then + echo -e "${FRED}Error: ${job_filepath} doesn't exist, or is not executable.${RS}" >&2; + exit 1; +fi + +log_msg() { + echo -e "[ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $*" >&2; +} + +# $1 Command to execute +RUN_ONCE() { + command="${1}"; + hash="$(echo "${command}" | sha256sum | cut -d- -f1)"; + + if [[ -z "${EXECUTE_HOSTNAME}" ]]; then + task_end 1 "Oops! The hostname to execute on wasn't found. This is probably a bug."; + fi + + log_msg "️🛡️ ${HC}[ ${EXECUTE_HOSTNAME} ]${RS} ${FBLE}RUN_ONCE${RS} ${command}"; + + if [[ "${INTERACTIVE}" == "true" ]]; then + echo "Enabling interactive mode"; + SSH_FLAGS="-t "; + fi + + echo " +if [[ ! -d /etc/remote-exec ]]; then sudo mkdir /etc/remote-exec; fi +if [[ ! -f /etc/remote-exec/${hash} ]]; then + echo \"[ \${HOSTNAME} ] [ \$(date -u +\"%Y-%m-%dT%H:%M:%SZ\") ] Executing once\" + ${command} + sudo touch /etc/remote-exec/${hash}; +fi +" | ssh ${SSH_FLAGS} -F "${ssh_configfile}" "${EXECUTE_HOSTNAME}" bash; + +} + +# $1 Command to execute +RUN() { + command="${1}"; + + if [[ -z "${EXECUTE_HOSTNAME}" ]]; then + task_end 1 "Oops! The hostname to execute on wasn't found. This is probably a bug."; + fi + + log_msg "🚏 ${HC}[ ${EXECUTE_HOSTNAME} ]${RS} ${FBLE}RUN${RS} ${command}"; + + if [[ "${INTERACTIVE}" == "true" ]]; then + echo "Enabling interactive mode"; + SSH_FLAGS="-t "; + fi + + echo "${command}" | ssh ${SSH_FLAGS} -F "${ssh_configfile}" "${EXECUTE_HOSTNAME}" bash; +} + +# $1 Filepath to script to copy & execute +SCRIPT() { + script_filepath="${1}"; + + remote_filepath="/tmp/$(basename ${script_filepath})"; + + if [[ -z "${EXECUTE_HOSTNAME}" ]]; then + task_end 1 "Oops! The hostname to execute on wasn't found. This is probably a bug."; + fi + + log_msg "🚂 ${HC}[ ${EXECUTE_HOSTNAME} ]${RS} ${FMAG}SCRIPT${RS} ${command}"; + + scp -F "${ssh_configfile}" "${script_filepath}" "${EXECUTE_HOSTNAME}:${remote_filepath}"; + echo -e "${remote_filepath}\nrm ${remote_filepath}" | ssh -F "${ssh_configfile}" "${EXECUTE_HOSTNAME}" bash; + echo "" +} + +# $1 The filepath to copy +# $2 The location to copy it to on the remote host +COPY() { + filepath_local="${1}"; + filepath_remote="${2}"; + + log_msg "🚚 ${HC}[ ${EXECUTE_HOSTNAME} ]${RS} ${FMAG}COPY${RS} ${filepath_local} → ${filepath_remote}"; + + if [[ ! -f "${filepath_local}" ]]; then + task_end 1 "[ ${HC}COPY${RS} ] Error: Couldn't find ${filepath_local} on disk (CWD is ${PWD})."; + fi + + if [[ -z "${EXECUTE_HOSTNAME}" ]]; then + task_end 1 "[ ${HC}COPY${RS} ] Oops! The hostname to copy to wasn't found. This is probably a bug."; + fi + + + scp -F "${ssh_configfile}" "${filepath_local}" "${EXECUTE_HOSTNAME}:${filepath_remote}"; +} + +# $1 job filepath +# $2 hostname +__do_execute() { + job_filepath="${1}"; + hostname="${2}"; + log_msg "${FBLE}${HC}⌛${RS} Starting on ${HC}${hostname}${RS}"; + + export EXECUTE_HOSTNAME="${hostname}"; + export JOBFILE_DIR; JOBFILE_DIR="$(dirname "${job_filepath}")"; + + #shellcheck disable=SC1090 + source "${job_filepath}"; + + log_msg "${FGRN}${HC}✔${RS} Finished on ${HC}${hostname}${RS}"; +} + +task_begin "Executing ${job_filepath}"; + +while read -r hostname; do + if [[ -z "${hostname}" ]]; then + continue; + fi + + if [[ "${do_parallel}" == "true" ]]; then + __do_execute "${job_filepath}" "${hostname}" & + else + __do_execute "${job_filepath}" "${hostname}"; + fi +done < <(echo "${hosts}"); + +wait diff --git a/hosts/clients-consul.txt b/hosts/clients-consul.txt new file mode 100644 index 0000000..e565d74 --- /dev/null +++ b/hosts/clients-consul.txt @@ -0,0 +1,2 @@ +cluster-balusha +cluster-silverleaf diff --git a/hosts/clients-nomad.txt b/hosts/clients-nomad.txt new file mode 100644 index 0000000..92e6683 --- /dev/null +++ b/hosts/clients-nomad.txt @@ -0,0 +1,2 @@ +cluster-sycadil +cluster-spatterling diff --git a/hosts/controller.txt b/hosts/controller.txt new file mode 100644 index 0000000..7c255f4 --- /dev/null +++ b/hosts/controller.txt @@ -0,0 +1 @@ +cluster-wopplefox diff --git a/hosts/servers-consul.txt b/hosts/servers-consul.txt new file mode 100644 index 0000000..92e6683 --- /dev/null +++ b/hosts/servers-consul.txt @@ -0,0 +1,2 @@ +cluster-sycadil +cluster-spatterling diff --git a/hosts/servers-nomad.txt b/hosts/servers-nomad.txt new file mode 100644 index 0000000..e565d74 --- /dev/null +++ b/hosts/servers-nomad.txt @@ -0,0 +1,2 @@ +cluster-balusha +cluster-silverleaf diff --git a/hosts/servers-vault.txt b/hosts/servers-vault.txt new file mode 100644 index 0000000..87c1630 --- /dev/null +++ b/hosts/servers-vault.txt @@ -0,0 +1 @@ +cluster-balusha diff --git a/hosts/workers.txt b/hosts/workers.txt new file mode 100644 index 0000000..af9c274 --- /dev/null +++ b/hosts/workers.txt @@ -0,0 +1,4 @@ +cluster-sycadil +cluster-spatterling +cluster-balusha +cluster-silverleaf diff --git a/jobs/apt-cache/apt-cache.job b/jobs/apt-cache/apt-cache.job new file mode 100755 index 0000000..79bdd3a --- /dev/null +++ b/jobs/apt-cache/apt-cache.job @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +RUN "echo 'Acquire::http { Proxy \"http://172.16.230.100:3142\"; }' | sudo tee /etc/apt/apt.conf.d/proxy" + +# Use http instead of https to allow caching +RUN "echo 'deb http://archive.raspberrypi.org/debian/ buster main ui' | sudo tee /etc/apt/sources.list.d/raspi.list"; + +RUN_ONCE "sudo rm /etc/apt/sources.list.d/raspi"; diff --git a/jobs/avahi-daemon/avahi-daemon.job b/jobs/avahi-daemon/avahi-daemon.job new file mode 100755 index 0000000..d06d327 --- /dev/null +++ b/jobs/avahi-daemon/avahi-daemon.job @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +RUN "sudo apt install avahi-daemon" diff --git a/jobs/bin-folder/bin-folder.job b/jobs/bin-folder/bin-folder.job new file mode 100755 index 0000000..c0710a9 --- /dev/null +++ b/jobs/bin-folder/bin-folder.job @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +RUN "sudo apt-get update"; +RUN "sudo apt-get install git"; +RUN "git clone https://gitlab.com/sbrl/bin.git" +RUN "cd \${HOME}/bin && GPG_SIGN_COMMITS=n ./envsetup"; diff --git a/jobs/collectd/collectd.conf b/jobs/collectd/collectd.conf new file mode 100644 index 0000000..9f396ad --- /dev/null +++ b/jobs/collectd/collectd.conf @@ -0,0 +1,59 @@ +Interval 300 + +LoadPlugin load +LoadPlugin thermal +LoadPlugin memory +LoadPlugin disk +LoadPlugin swap +LoadPlugin uptime +#LoadPlugin smart + +LoadPlugin df +LoadPlugin irq +LoadPlugin contextswitch +LoadPlugin interface +LoadPlugin ping +#LoadPlugin nginx +#LoadPlugin ntpd + +LoadPlugin exec + +LoadPlugin processes + +LoadPlugin network + + + MountPoint "/" + MountPoint "/boot" + + + + Interface "eth0" + IgnoreSelected false + + + + Host "elessar.mooncarrot.space" + Host "wopplefox.mooncarrot.space" + Host "ubuntu.mirrors.ovh.net" + #Host "s3-eu-west-1.amazonaws.com" + #Host "github.com" + #Host "api.backblazeb2.com" + + +# +# Host "localhost" +# Port "123" +# + + + CollectMemoryMaps false + + + + + SecurityLevel "Encrypt" + Username "cluster" + Password "{{{PASSWORD}}}" + + diff --git a/jobs/collectd/collectd.job b/jobs/collectd/collectd.job new file mode 100755 index 0000000..e7e69a0 --- /dev/null +++ b/jobs/collectd/collectd.job @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +if [[ -z "${COLLECTD_PW}" ]]; then + echo "Error: COLLECTD_PW environment variable is not set"; + exit 1; +fi + +RUN "sudo apt-get install -y collectd liboping0"; + +tmpfile_collectd="$(mktemp --tmpdir "collectd-XXXXXXX")"; + +chmod 0600 "${tmpfile_collectd}"; +sed -e "s/{{{PASSWORD}}}/${COLLECTD_PW}/g" <"${JOBFILE_DIR}/collectd.conf" >"${tmpfile_collectd}"; +unset COLLECTD_PW; + +COPY "${tmpfile_collectd}" "/tmp/collectd.conf"; + +rm "${tmpfile_collectd}"; + +RUN "sudo chown root:root /tmp/collectd.conf"; +RUN "sudo chmod 0600 /tmp/collectd.conf"; +RUN "sudo mv /tmp/collectd.conf /etc/collectd/collectd.conf"; + +RUN "sudo systemctl restart collectd.service"; +RUN "sudo systemctl enable collectd.service"; diff --git a/jobs/config-git/config-git.job b/jobs/config-git/config-git.job new file mode 100755 index 0000000..a2ca806 --- /dev/null +++ b/jobs/config-git/config-git.job @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +RUN "sudo apt-get install --yes git openssh-client"; + +RUN 'ssh -T git@git.starbeamrainbowlabs.com || ssh-keyscan -H git.starbeamrainbowlabs.com >>$HOME/.ssh/known_hosts'; +RUN 'sudo -E ssh -T git@git.starbeamrainbowlabs.com || ssh-keyscan -H git.starbeamrainbowlabs.com | sudo tee -a /root/.ssh/known_hosts'; + +RUN "[[ -d /etc/cluster-config ]] && { cd /etc/cluster-config; sudo -E git pull; } || sudo -E git clone git@git.starbeamrainbowlabs.com:sbrl/cluster-config.git /etc/cluster-config"; diff --git a/jobs/consul-client/consul-client.job b/jobs/consul-client/consul-client.job new file mode 100755 index 0000000..82f05c9 --- /dev/null +++ b/jobs/consul-client/consul-client.job @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +SCRIPT "jobs/consul-server/common.sh"; + +COPY "../consul/client.hcl" "/tmp/client.hcl" + +RUN "sudo mv /tmp/client.hcl /etc/consul/client.hcl"; +RUN "sudo chown root:root /etc/consul/client.hcl"; +RUN "sudo apt-get update"; +RUN "sudo apt-get install --yes hashicorp-consul-systemd-client"; + +# Do we need these as a client? I'm guessing not....? +# If we run into issues, we'll unblock them. +# RUN "sudo ufw allow 8301 comment consul-serf-lan"; +# # RUN "sudo ufw allow 8300/tcp comment consul-rpc"; +# RUN "sudo ufw allow 8600 comment consul-dns"; + +# Commented out since we're now running over wgoverlay & allowing all traffic on that interface because dynamic ports +# Delete old ufw rules +RUN "sudo ufw delete allow 8301 comment consul-serf-lan"; +RUN "sudo ufw delete allow 8300/tcp comment consul-rpc"; +RUN "sudo ufw delete allow 8600 comment consul-dns"; + +RUN "sudo systemctl enable consul.service"; +RUN "sudo systemctl restart consul.service"; diff --git a/jobs/consul-server/common.sh b/jobs/consul-server/common.sh new file mode 100755 index 0000000..75cac89 --- /dev/null +++ b/jobs/consul-server/common.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +log_msg() { + echo -e "[ ${HOSTNAME} ] [ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $@" >&2; +} + + +log_msg "Installing consul"; +sudo apt-get update; +sudo apt-get install --yes hashicorp-consul; + +log_msg "consul installed successfully. Version:"; +consul --version; + +log_msg "Creating directories"; +if [[ ! -d "/etc/consul" ]]; then + sudo mkdir /etc/consul; + sudo chown root:root /etc/consul; +fi + +if [[ ! -d "/srv/consul" ]]; then + sudo mkdir /srv/consul; + sudo chown root:root /srv/consul; + sudo chmod 0750 /srv/consul; +fi diff --git a/jobs/consul-server/consul-server.job b/jobs/consul-server/consul-server.job new file mode 100755 index 0000000..640710e --- /dev/null +++ b/jobs/consul-server/consul-server.job @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +SCRIPT "${JOBFILE_DIR}/common.sh"; + +COPY "../consul/server.hcl" "/tmp/server.hcl" + +RUN "sudo mv /tmp/server.hcl /etc/consul/server.hcl"; +RUN "sudo chown root:root /etc/consul/server.hcl"; +RUN "sudo apt-get update"; +RUN "sudo apt-get install --yes hashicorp-consul-systemd-server"; + +# TODO: Restrict these to the wesher subnet, and add 8500/tcp (the HTTP API) +# TODO: Do that for Nomad too +# RUN "sudo ufw allow 8301 comment consul-serf-lan"; +# RUN "sudo ufw allow 8300/tcp comment consul-rpc"; +# RUN "sudo ufw allow 8600 comment consul-dns"; + +# Commented out since we're now running over wgoverlay & allowing all traffic on that interface because dynamic ports +# Delete old ufw rules +RUN "sudo ufw delete allow 8301 comment consul-serf-lan"; +RUN "sudo ufw delete allow 8300/tcp comment consul-rpc"; +RUN "sudo ufw delete allow 8600 comment consul-dns"; + +RUN "sudo systemctl enable consul.service"; +RUN "sudo systemctl restart consul.service"; diff --git a/jobs/dns/dns.job b/jobs/dns/dns.job new file mode 100755 index 0000000..4547d6b --- /dev/null +++ b/jobs/dns/dns.job @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +SCRIPT "${JOBFILE_DIR}/dns.sh"; diff --git a/jobs/dns/dns.sh b/jobs/dns/dns.sh new file mode 100755 index 0000000..af15c5d --- /dev/null +++ b/jobs/dns/dns.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Usage: +# configLine [searchPattern] [replaceLine] [filePath] +# Ref https://stackoverflow.com/a/54909102/1460422 +config_line() { + local OLD_LINE_PATTERN=$1; shift + local NEW_LINE=$1; shift + local FILE=$1 + local NEW=$(echo "${NEW_LINE}" | sed 's/\//\\\//g') + touch "${FILE}" + sed -i '/'"${OLD_LINE_PATTERN}"'/{s/.*/'"${NEW}"'/;h};${x;/./{x;q100};x}' "${FILE}" + if [[ $? -ne 100 ]] && [[ ${NEW_LINE} != '' ]] + then + echo "${NEW_LINE}" >> "${FILE}" + fi +} + +log_msg() { + echo -e "[ ${HOSTNAME} ] [ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $@" >&2; +} + +# Old approach - we've learned we can override /etc/network/interfaces via /etc/dhcp/dhclient.conf +# config_line "dns-nameservers" "dns-nameservers 172.16.230.100" "/etc/network/interfaces"; + +completed_file="/etc/dhcp/sbrl-dns-configured"; + +log_msg "Configuring DNS nameserver"; +if [[ ! -f "${completed_file}" ]]; then + log_msg "DNS nameserver not yet configured, appending to /etc/dhcp/dhclient.conf"; + echo "interface \"eth0\" { + supersede domain-name-servers 172.16.230.100; +}" | sudo tee -a /etc/dhcp/dhclient.conf; + sudo touch "${completed_file}"; + + log_msg "Complete, this machine needs a reboot to activate the new config directives"; +else + log_msg "DNS nameserver config written already, no changes made"; +fi diff --git a/jobs/docker/docker.job b/jobs/docker/docker.job new file mode 100755 index 0000000..c68121e --- /dev/null +++ b/jobs/docker/docker.job @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Fix apt update errors +# Ref https://blog.samcater.com/fix-workaround-rpi4-docker-libseccomp2-docker-20/ +RUN "sudo apt-get install libseccomp2 -t buster-backports"; + +# RUN "curl -fsSL https://get.docker.com -o /tmp/get-docker.sh"; +# RUN "chmod +x /tmp/get-docker.sh"; +# RUN "sudo sh /tmp/get-docker.sh"; +RUN 'echo "deb [arch=$(dpkg --print-architecture)] http://download.docker.com/linux/raspbian buster stable" | sudo tee /etc/apt/sources.list.d/docker.list'; +RUN "sudo apt-get update"; +RUN "sudo apt-get install --yes docker-ce"; +RUN "if [[ ! -e \"/etc/docker/daemon.json\" ]]; then echo '{\"insecure-registries\":[\"registry.service.mooncarrot.space:5000\"]}' | sudo tee /etc/docker/daemon.json; fi"; +RUN "sudo systemctl reload docker.service"; diff --git a/jobs/install-utils-python/install-utils-python.job b/jobs/install-utils-python/install-utils-python.job new file mode 100755 index 0000000..fb7472a --- /dev/null +++ b/jobs/install-utils-python/install-utils-python.job @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +RUN "sudo apt-get install python3-pip"; + +RUN "sudo pip3 install setuptools"; +RUN "sudo pip3 install quarry"; diff --git a/jobs/install-utils/install-utils.job b/jobs/install-utils/install-utils.job new file mode 100755 index 0000000..ea7043a --- /dev/null +++ b/jobs/install-utils/install-utils.job @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +RUN "sudo apt-get install --no-install-recommends --yes git net-tools dnsutils uptimed software-properties-common iftop iotop pv screen netcat f3 less optipng jpegoptim lsof strace ddrescue ffmpeg lnav jq"; diff --git a/jobs/laminar-sshkey/laminar-sshkey.job b/jobs/laminar-sshkey/laminar-sshkey.job new file mode 100755 index 0000000..f52ff36 --- /dev/null +++ b/jobs/laminar-sshkey/laminar-sshkey.job @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +sshkey="$(mktemp --tmpdir "laminarci-sshkey-pub-XXXXXXX")"; +echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICzYSWsGbWHh+cDheHJSKlCrxYDLwS6SKwVMV9SAaGsO laminar-ci@elessar" >"${sshkey}"; + +upload_target="/tmp/laminarci-sshkey-pub"; + +COPY "${sshkey}" "${upload_target}"; + +RUN "sudo mkdir -p /root/.ssh"; +RUN_ONCE "cat '${upload_target}' | sudo tee /root/.ssh/authorized_keys"; +RUN "rm '${upload_target}'"; + +# Set permissions +RUN "sudo chown -R root:root /root/.ssh"; +RUN "sudo chmod 0600 /root/.ssh/authorized_keys"; diff --git a/jobs/maintenance/maintenance.job b/jobs/maintenance/maintenance.job new file mode 100755 index 0000000..3f10e6c --- /dev/null +++ b/jobs/maintenance/maintenance.job @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Remove any unused Docker images +RUN "sudo docker image prune --force"; diff --git a/jobs/nfs/cluster-shared-nfs b/jobs/nfs/cluster-shared-nfs new file mode 100755 index 0000000..4736a5c --- /dev/null +++ b/jobs/nfs/cluster-shared-nfs @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +if ! mountpoint -q /mnt/shared; then mount /mnt/shared; fi +if ! mountpoint -q /mnt/elfstone; then mount /mnt/elfstone; fi +if ! mountpoint -q /mnt/elessar-music; then mount /mnt/elessar-music; fi +if ! mountpoint -q /mnt/elessar-syncthing; then mount /mnt/elessar-syncthing; fi diff --git a/jobs/nfs/nfs-setup.sh b/jobs/nfs/nfs-setup.sh new file mode 100755 index 0000000..ede3bfe --- /dev/null +++ b/jobs/nfs/nfs-setup.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +echo "[job/nfs] Checking NFS mounts" >&2; + +# _netdev The filesystem resides on a device that requires network access (used to prevent the system from attempting to mount these filesystems until the network has been enabled on the system) +# nofail Do not report errors for this device if it does not exist. +# auto Auto-mount on boot +# noatime No last-access time +# tcp Use tcp, not udp +# bg The system won't block during boot until it is able to mount the filesystem, but won't give up on trying to mount it either. +# timeo=VALUE The time in deciseconds (tenths of a second) the NFS client waits for a response before it retries an NFS request. +# retrans=n The number of times the NFS client retries a request before it attempts further recovery action. + + +if [[ ! -d /mnt/shared ]]; then + echo "[job/nfs] Creating mount for shared cluster data" >&2; + sudo mkdir -p /mnt/shared; + echo 'magicbag.node.mooncarrot.space:/mnt/elfstone2/cluster /mnt/shared nfs auto,nofail,noatime,_netdev,tcp,bg,timeo=50,retrans=5 0 0' | sudo tee -a /etc/fstab; + sudo mount /mnt/shared; +fi + + +if [[ ! -d /mnt/elfstone ]]; then + echo "[job/nfs] Creating mount for elfstone" >&2; + sudo mkdir -p /mnt/elfstone; + echo 'magicbag.node.mooncarrot.space:/mnt/elfstone2/main /mnt/elfstone nfs auto,nofail,noatime,_netdev,tcp,bg,timeo=50,retrans=5 0 0' | sudo tee -a /etc/fstab; + sudo mount /mnt/elfstone; +fi + + +if [[ ! -d /mnt/elessar-music ]]; then + echo "[job/nfs] Creating mount for elessar-music" >&2; + sudo mkdir -p /mnt/elessar-music; + echo 'magicbag.node.mooncarrot.space:/mnt/elfstone2/syncthing/Music /mnt/elessar-music nfs auto,nofail,_netdev,noatime,tcp,bg,timeo=50,retrans=5 0 0' | sudo tee -a /etc/fstab; + sudo mount /mnt/elessar-music; +fi + + +if [[ ! -d /mnt/elessar-syncthing ]]; then + echo "[job/nfs] Creating mount for elessar-syncthing" >&2; + sudo mkdir -p /mnt/elessar-syncthing; + echo 'magicbag.node.mooncarrot.space:/mnt/elfstone2/syncthing /mnt/elessar-syncthing nfs auto,nofail,_netdev,noatime,tcp,bg,timeo=50,retrans=5 0 0' | sudo tee -a /etc/fstab; + sudo mount /mnt/elessar-syncthing; +fi + +echo "[job/nfs] Complete" >&2; diff --git a/jobs/nfs/nfs.job b/jobs/nfs/nfs.job new file mode 100755 index 0000000..bc6d50f --- /dev/null +++ b/jobs/nfs/nfs.job @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +RUN "sudo apt-get update"; +RUN "sudo apt-get install --yes nfs-common"; + +SCRIPT "${JOBFILE_DIR}/nfs-setup.sh" + +COPY "${JOBFILE_DIR}/cluster-shared-nfs" "/tmp/cluster-shared-nfs"; +RUN "sudo mv /tmp/cluster-shared-nfs /etc/network/if-up.d/cluster-shared-nfs"; +RUN "sudo chown root:root /etc/network/if-up.d/cluster-shared-nfs"; +RUN "sudo chmod +x /etc/network/if-up.d/cluster-shared-nfs" diff --git a/jobs/nomad-both/common.sh b/jobs/nomad-both/common.sh new file mode 100755 index 0000000..5d420fb --- /dev/null +++ b/jobs/nomad-both/common.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +log_msg() { + echo -e "[ ${HOSTNAME} ] [ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $*" >&2; +} + + +log_msg "Installing nomad"; +sudo apt-get update; +sudo apt-get install --yes hashicorp-nomad; + +log_msg "nomad installed successfully. Version:"; +nomad --version; + +log_msg "Creating directories"; +if [[ ! -d "/etc/nomad" ]]; then + sudo mkdir /etc/nomad; + sudo chown root:root /etc/nomad; +fi + +if [[ ! -d "/srv/nomad" ]]; then + sudo mkdir /srv/nomad; + sudo chown root:root /srv/nomad; + sudo chmod 0750 /srv/nomad; +fi + +check_cgroups_memory() { + log_msg "Checking memory cgroups"; + + cgroups_enabled="$(awk '/memory/ { print $2 }' < /proc/cgroups)"; + + if [[ "${cgroups_enabled}" -ne 0 ]]; then + log_msg "memory cgroups already enabled"; + return 0; + fi + + + filepath_cmdline="/boot/cmdline.txt"; + if [[ ! -e "${filepath_cmdline}" ]]; then + filepath_cmdline="/boot/firmware/cmdline.txt"; + fi + if [[ ! -e "${filepath_cmdline}" ]]; then + log_msg "Failed to find cmdline.txt; can't check for cgroups"; + return 1; + fi + + if grep -q cgroup_enable=memory /boot/cmdline.txt; then + log_msg "memory cgroups already present in cmdline.txt, a reboot is required to apply the update"; + return 0; + fi + + log_msg "memory cgroups not present in cmdline.txt, enabling...."; + (tr -d '\n' <"${filepath_cmdline}" && echo " cgroup_enable=memory cgroup_memory=1") | sudo tee "${filepath_cmdline}.new"; + + sudo mv "${filepath_cmdline}" "${filepath_cmdline}.old-$(date +"%Y-%m-%d")"; + sudo mv "${filepath_cmdline}.new" "${filepath_cmdline}"; + + log_msg "New contents of cmdline.txt:"; + cat "${filepath_cmdline}"; + log_msg "A reboot is required to apply the changes."; +} + +check_cgroups_memory; diff --git a/jobs/nomad-both/nomad-both.job b/jobs/nomad-both/nomad-both.job new file mode 100755 index 0000000..0d7c9f1 --- /dev/null +++ b/jobs/nomad-both/nomad-both.job @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +SCRIPT "${JOBFILE_DIR}/common.sh"; + +COPY "../nomad/both.hcl" "/tmp/both.hcl" + +RUN "sudo mv /tmp/both.hcl /etc/nomad/both.hcl"; +RUN "sudo chown root:root /etc/nomad/both.hcl"; + +RUN "sudo apt-get install --yes hashicorp-nomad-systemd-both"; + + +# Commented out since we're now running over wgoverlay & allowing all traffic on that interface because dynamic ports +# RUN "sudo ufw allow 4646/tcp comment nomad-http"; +# RUN "sudo ufw allow 4647/tcp comment nomad-rpc"; +# RUN "sudo ufw allow 4648/tcp comment nomad-serf"; +RUN "sudo ufw delete allow 4646/tcp comment nomad-http"; +RUN "sudo ufw delete allow 4647/tcp comment nomad-rpc"; +RUN "sudo ufw delete allow 4648/tcp comment nomad-serf"; + +RUN "sudo systemctl enable nomad.service"; +RUN "sudo systemctl restart nomad.service"; diff --git a/jobs/nomad-client/nomad-client.job b/jobs/nomad-client/nomad-client.job new file mode 100755 index 0000000..6e1af52 --- /dev/null +++ b/jobs/nomad-client/nomad-client.job @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +SCRIPT "jobs/nomad-both/common.sh"; + +COPY "../nomad/client.hcl" "/tmp/client.hcl" + +RUN "sudo mv /tmp/client.hcl /etc/nomad/client.hcl"; +RUN "sudo chown root:root /etc/nomad/client.hcl"; + +RUN "sudo apt-get install --yes hashicorp-nomad-systemd-client"; + +# Commented out since we're now running over wgoverlay & allowing all traffic on that interface because dynamic ports +# RUN "sudo ufw allow 4646/tcp comment nomad-http"; +# RUN "sudo ufw allow 4647/tcp comment nomad-rpc"; +# RUN "sudo ufw allow 4648/tcp comment nomad-serf"; +RUN "sudo ufw delete allow 4646/tcp comment nomad-http"; +RUN "sudo ufw delete allow 4647/tcp comment nomad-rpc"; +RUN "sudo ufw delete allow 4648/tcp comment nomad-serf"; + +RUN "sudo systemctl enable nomad.service"; +RUN "sudo systemctl restart nomad.service"; diff --git a/jobs/preconfigure/misc b/jobs/preconfigure/misc new file mode 100644 index 0000000..e69de29 diff --git a/jobs/preconfigure/preconfigure.job b/jobs/preconfigure/preconfigure.job new file mode 100755 index 0000000..082d92a --- /dev/null +++ b/jobs/preconfigure/preconfigure.job @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +RUN "sudo apt-get install -y logrotate" + +COPY "${JOBFILE_DIR}/misc" "/tmp/misc" +RUN "sudo mv /tmp/misc /etc/logrotate.d/misc" +RUN "sudo chown root:root /etc/logrotate.d/misc" + +# Fix logrotate issue +RUN "sudo chmod 0750 /var/log" +RUN "sudo systemctl restart logrotate" diff --git a/jobs/sbrl-apt-repo/sbrl-apt-repo.job b/jobs/sbrl-apt-repo/sbrl-apt-repo.job new file mode 100755 index 0000000..a55c6b5 --- /dev/null +++ b/jobs/sbrl-apt-repo/sbrl-apt-repo.job @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +RUN "echo 'deb http://apt.starbeamrainbowlabs.com/ ./ # apt.starbeamrainbowlabs.com' | sudo tee /etc/apt/sources.list.d/sbrl.list"; + +RUN "sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D48D801C6A66A5D8"; + +RUN "sudo apt-get update"; diff --git a/jobs/ssh-preconnect/ssh-preconnect.job b/jobs/ssh-preconnect/ssh-preconnect.job new file mode 100755 index 0000000..a67fd9c --- /dev/null +++ b/jobs/ssh-preconnect/ssh-preconnect.job @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +RUN 'ssh -T git@git.starbeamrainbowlabs.com || ssh-keyscan -H git.starbeamrainbowlabs.com >>$HOME/.ssh/known_hosts'; +RUN 'sudo -E ssh -T git@git.starbeamrainbowlabs.com || ssh-keyscan -H git.starbeamrainbowlabs.com | sudo tee -a /root/.ssh/known_hosts'; diff --git a/jobs/test/test.job b/jobs/test/test.job new file mode 100755 index 0000000..b3c2c1e --- /dev/null +++ b/jobs/test/test.job @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +RUN "uptime"; + +COPY "${JOBFILE_DIR}/testfile.txt" "/tmp"; + +RUN "rm /tmp/testfile.txt"; diff --git a/jobs/test/testfile.txt b/jobs/test/testfile.txt new file mode 100644 index 0000000..6c09a71 --- /dev/null +++ b/jobs/test/testfile.txt @@ -0,0 +1 @@ +it works! diff --git a/jobs/ufw/ufw.job b/jobs/ufw/ufw.job new file mode 100755 index 0000000..f0beb2f --- /dev/null +++ b/jobs/ufw/ufw.job @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +RUN "sudo apt-get update"; + +RUN "sudo apt-get install --yes ufw"; + +RUN "sudo ufw allow 22/tcp comment ssh"; + +RUN "echo 'y' | sudo ufw enable"; diff --git a/jobs/update-system/update-system.job b/jobs/update-system/update-system.job new file mode 100755 index 0000000..4f68833 --- /dev/null +++ b/jobs/update-system/update-system.job @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +SCRIPT "${JOBFILE_DIR}/update-system.sh"; diff --git a/jobs/update-system/update-system.sh b/jobs/update-system/update-system.sh new file mode 100755 index 0000000..6c41278 --- /dev/null +++ b/jobs/update-system/update-system.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +log_msg() { + echo -e "[ ${HOSTNAME} ] [ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $@" >&2; +} + +log_msg "Starting update"; +sudo apt-get update; + +log_msg "Starting upgrade"; +sudo apt-get dist-upgrade --yes; + +log_msg "Starting autoremove"; +sudo apt-get autoremove --yes; + +log_msg "Starting autoclean"; +sudo apt-get autoclean; diff --git a/jobs/vault/vault.job b/jobs/vault/vault.job new file mode 100755 index 0000000..0032f75 --- /dev/null +++ b/jobs/vault/vault.job @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +RUN "sudo apt-get install -y hashicorp-vault hashicorp-vault-systemd"; + +COPY "../vault/server.hcl" "/tmp/vault-server.hcl"; +RUN "sudo chown root:root /tmp/vault-server.hcl"; +RUN "sudo mkdir -p /etc/vault"; +RUN "sudo mv /tmp/vault-server.hcl /etc/vault/server.hcl"; + +RUN "sudo systemctl enable vault.service"; +RUN "sudo systemctl start vault.service"; diff --git a/jobs/wesher/wesher.job b/jobs/wesher/wesher.job new file mode 100755 index 0000000..1b76d14 --- /dev/null +++ b/jobs/wesher/wesher.job @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +echo "Don't forget to run the wireguard job first"; + +# Sort out log rotation +COPY "${JOBFILE_DIR}/wesher.logrotate" "/tmp/wesher.logrotate" +RUN "sudo mv /tmp/wesher.logrotate /etc/logrotate.d/wesher" +RUN "sudo chown root:root /etc/logrotate.d/wesher" + +RUN "sudo apt-get update"; + +RUN "sudo apt-get install --yes wesher wesher-systemd"; + +RUN "sudo ufw allow 7946 comment wesher-gossip"; +RUN "sudo ufw allow 51820/udp comment wesher-wireguard"; + +# Allow all traffic from the VPN; we don't actually know what ports Nomad will use +RUN "sudo ufw allow in on wgoverlay"; + +# Activate the new logging system +RUN "sudo systemctl restart logrotate" + +# See the Keepass entry in our database for copy-paste commands to join nodes. +# Basically: +# sudo wesher --cluster-key CLUSTER_KEY_HERE --join 172.16.230.100 --overlay-net 172.31.250.0/16 --log-level info +# sudo systemctl enable --now wesher.service +# sudo systemctl status wesher.service diff --git a/jobs/wesher/wesher.logrotate b/jobs/wesher/wesher.logrotate new file mode 100644 index 0000000..903b5d0 --- /dev/null +++ b/jobs/wesher/wesher.logrotate @@ -0,0 +1,12 @@ +/var/log/wesher/*.log { + rotate 12 + weekly + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + invoke-rc.d rsyslog rotate >/dev/null 2>&1 || true + endscript +} diff --git a/jobs/wireguard/wireguard.job b/jobs/wireguard/wireguard.job new file mode 100755 index 0000000..0d860ab --- /dev/null +++ b/jobs/wireguard/wireguard.job @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +RUN "sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 04EE7237B7D453EC 648ACFD622F3D138"; + +RUN "echo 'deb http://httpredir.debian.org/debian buster-backports main contrib non-free' | sudo tee /etc/apt/sources.list.d/debian-backports.list"; + +RUN "sudo apt-get update"; + +RUN "sudo apt-get install --yes raspberrypi-kernel-headers"; + +RUN "sudo apt-get install --yes wireguard"; diff --git a/list-hostgroups b/list-hostgroups new file mode 100755 index 0000000..c56c2e8 --- /dev/null +++ b/list-hostgroups @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Make sure the current directory is the location of this script to simplify matters +cd "$(dirname "$(readlink -f "$0")")"; + +find hosts -type f -iname '*.txt' | xargs -n1 basename --suffix=.txt | sort diff --git a/list-jobs b/list-jobs new file mode 100755 index 0000000..0375f58 --- /dev/null +++ b/list-jobs @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Make sure the current directory is the location of this script to simplify matters +cd "$(dirname "$(readlink -f "$0")")"; + +ls ./jobs/ diff --git a/new-job b/new-job new file mode 100755 index 0000000..0143aa4 --- /dev/null +++ b/new-job @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Make sure the current directory is the location of this script to simplify matters +cd "$(dirname "$(readlink -f "$0")")"; + +job_name="$1"; + +if [[ -z "${job_name}" ]]; then + echo "Usage:"; + echo " ./new-job {job_name}"; + exit; +fi + +job_filename="./jobs/${job_name}/${job_name}.job"; + +mkdir -p "$(dirname "${job_filename}")"; + +echo -e "#!/usr/bin/env bash\n" >"${job_filename}"; +chmod +x "${job_filename}"; diff --git a/ssh-config b/ssh-config new file mode 100644 index 0000000..1fad6e0 --- /dev/null +++ b/ssh-config @@ -0,0 +1,46 @@ +ServerAliveInterval 120 + +Host * + ControlMaster auto + ControlPath /tmp/%r@%h:%p + + +Host cluster-wopplefox + Hostname 172.16.230.100 + Port 22 + + User root + IdentityFile ~/.ssh/id_ed25519 + ForwardAgent yes +Host cluster-sycadil + Hostname 172.16.230.102 + Port 22 + + User dietpi + IdentityFile ~/.ssh/id_ed25519 + ForwardAgent yes +Host cluster-spatterling + Hostname 172.16.230.101 + Port 22 + + User dietpi + IdentityFile ~/.ssh/id_ed25519 + ForwardAgent yes +Host cluster-balusha + Hostname 172.16.230.103 + Port 22 + + User dietpi + IdentityFile ~/.ssh/id_ed25519 + ForwardAgent yes +Host cluster-silverleaf + # braskin + # toko + # *silverleaf* + # elligon: router + Hostname 172.16.230.104 + Port 22 + + User dietpi + IdentityFile ~/.ssh/id_ed25519 + ForwardAgent yes