Import from https://git.starbeamrainbowlabs.com/sbrl/cluster-config/

execute

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+set -e;
+
+# Interesting chars:
+# 🛰 - Satellite - looks very cool indeed in Firefox
+
+lantern_path="../lantern-build-engine";
+if [[ -z "${do_parallel}" ]]; then
+	do_parallel="true";
+fi
+
+###############################################################################
+
+# Make sure the current directory is the location of this script to simplify matters
+cd "$(dirname "$(readlink -f "$0")")";
+
+# Check out the lantern git submodule if needed
+if [ ! -f "${lantern_path}/lantern.sh" ]; then git submodule update --init "${lantern_path}"; fi
+
+#shellcheck disable=SC1090
+source "${lantern_path}/lantern.sh";
+
+###############################################################################
+
+if [[ -z "${2}" ]]; then
+	echo "Usage:";
+	echo "	./execute {{path/to/jobfile}} {{path/to/hostsfile_a}} {{path/to/hostsfile_n}}";
+	exit 1;
+fi
+
+job_filepath="$1";
+if [[ ! -f "${job_filepath}" ]]; then
+	job_filepath="jobs/${job_filepath}/${job_filepath}.job";
+fi
+
+shift;
+
+hosts="";
+while [[ ! -z "${1}" ]]; do
+	hosts_filename="${1}";
+	if [[ ! -f "${hosts_filename}" ]]; then
+		hosts_filename="hosts/${hosts_filename}.txt";
+	fi
+	
+	hosts="$(echo "${hosts}" && cat "${hosts_filename}")";
+	shift;
+done
+
+ssh_configfile="./ssh-config";
+
+if [[ ! -x "${job_filepath}" ]]; then
+	echo -e "${FRED}Error: ${job_filepath} doesn't exist, or is not executable.${RS}" >&2;
+	exit 1;
+fi
+
+log_msg() {
+	echo -e "[ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $*" >&2;
+}
+
+# $1	Command to execute
+RUN_ONCE() {
+	command="${1}";
+	hash="$(echo "${command}" | sha256sum | cut -d- -f1)";
+	
+	if [[ -z "${EXECUTE_HOSTNAME}" ]]; then
+		task_end 1 "Oops! The hostname to execute on wasn't found. This is probably a bug.";
+	fi
+	
+	log_msg "️🛡️ ${HC}[ ${EXECUTE_HOSTNAME} ]${RS} ${FBLE}RUN_ONCE${RS} ${command}";
+	
+	if [[ "${INTERACTIVE}" == "true" ]]; then
+		echo "Enabling interactive mode";
+		SSH_FLAGS="-t ";
+	fi
+	
+	echo "
+if [[ ! -d /etc/remote-exec ]]; then sudo mkdir /etc/remote-exec; fi
+if [[ ! -f /etc/remote-exec/${hash} ]]; then
+	echo \"[ \${HOSTNAME} ] [ \$(date -u +\"%Y-%m-%dT%H:%M:%SZ\") ] Executing once\"
+	${command}
+	sudo touch /etc/remote-exec/${hash};
+fi
+" | ssh ${SSH_FLAGS} -F "${ssh_configfile}" "${EXECUTE_HOSTNAME}" bash;
+	
+}
+
+# $1	Command to execute
+RUN() {
+	command="${1}";
+	
+	if [[ -z "${EXECUTE_HOSTNAME}" ]]; then
+		task_end 1 "Oops! The hostname to execute on wasn't found. This is probably a bug.";
+	fi
+	
+	log_msg "🚏 ${HC}[ ${EXECUTE_HOSTNAME} ]${RS} ${FBLE}RUN${RS} ${command}";
+	
+	if [[ "${INTERACTIVE}" == "true" ]]; then
+		echo "Enabling interactive mode";
+		SSH_FLAGS="-t ";
+	fi
+		
+	echo "${command}" | ssh ${SSH_FLAGS} -F "${ssh_configfile}" "${EXECUTE_HOSTNAME}" bash;
+}
+
+# $1	Filepath to script to copy & execute
+SCRIPT() {
+	script_filepath="${1}";
+	
+	remote_filepath="/tmp/$(basename ${script_filepath})";
+	
+	if [[ -z "${EXECUTE_HOSTNAME}" ]]; then
+		task_end 1 "Oops! The hostname to execute on wasn't found. This is probably a bug.";
+	fi
+	
+	log_msg "🚂 ${HC}[ ${EXECUTE_HOSTNAME} ]${RS} ${FMAG}SCRIPT${RS} ${command}";
+	
+	scp -F "${ssh_configfile}" "${script_filepath}" "${EXECUTE_HOSTNAME}:${remote_filepath}";
+	echo -e "${remote_filepath}\nrm ${remote_filepath}" | ssh -F "${ssh_configfile}" "${EXECUTE_HOSTNAME}" bash;
+	echo ""
+}
+
+# $1	The filepath to copy
+# $2	The location to copy it to on the remote host
+COPY() {
+	filepath_local="${1}";
+	filepath_remote="${2}";
+	
+	log_msg "🚚 ${HC}[ ${EXECUTE_HOSTNAME} ]${RS} ${FMAG}COPY${RS} ${filepath_local} → ${filepath_remote}";
+	
+	if [[ ! -f "${filepath_local}" ]]; then
+		task_end 1 "[ ${HC}COPY${RS} ] Error: Couldn't find ${filepath_local} on disk (CWD is ${PWD}).";
+	fi
+	
+	if [[ -z "${EXECUTE_HOSTNAME}" ]]; then
+		task_end 1 "[ ${HC}COPY${RS} ] Oops! The hostname to copy to wasn't found. This is probably a bug.";
+	fi
+	
+	
+	scp -F "${ssh_configfile}" "${filepath_local}" "${EXECUTE_HOSTNAME}:${filepath_remote}";
+}
+
+# $1	job filepath
+# $2	hostname
+__do_execute() {
+	job_filepath="${1}";
+	hostname="${2}";
+	log_msg "${FBLE}${HC}⌛${RS} Starting on ${HC}${hostname}${RS}";
+	
+	export EXECUTE_HOSTNAME="${hostname}";
+	export JOBFILE_DIR; JOBFILE_DIR="$(dirname "${job_filepath}")";
+	
+	#shellcheck disable=SC1090
+	source "${job_filepath}";
+	
+	log_msg "${FGRN}${HC}✔${RS} Finished on ${HC}${hostname}${RS}";
+}
+
+task_begin "Executing ${job_filepath}";
+
+while read -r hostname; do
+	if [[ -z "${hostname}" ]]; then
+		continue;
+	fi
+	
+	if [[ "${do_parallel}" == "true" ]]; then
+		__do_execute "${job_filepath}" "${hostname}" &
+	else
+		__do_execute "${job_filepath}" "${hostname}";
+	fi
+done < <(echo "${hosts}");
+
+wait

hosts/clients-consul.txt

@@ -0,0 +1,2 @@
+cluster-balusha
+cluster-silverleaf

hosts/clients-nomad.txt

@@ -0,0 +1,2 @@
+cluster-sycadil
+cluster-spatterling

hosts/controller.txt

@@ -0,0 +1 @@
+cluster-wopplefox

hosts/servers-consul.txt

@@ -0,0 +1,2 @@
+cluster-sycadil
+cluster-spatterling

hosts/servers-nomad.txt

@@ -0,0 +1,2 @@
+cluster-balusha
+cluster-silverleaf

hosts/servers-vault.txt

@@ -0,0 +1 @@
+cluster-balusha

hosts/workers.txt

@@ -0,0 +1,4 @@
+cluster-sycadil
+cluster-spatterling
+cluster-balusha
+cluster-silverleaf

jobs/apt-cache/apt-cache.job

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+RUN "echo 'Acquire::http { Proxy \"http://172.16.230.100:3142\"; }' | sudo tee /etc/apt/apt.conf.d/proxy"
+
+# Use http instead of https to allow caching
+RUN "echo 'deb http://archive.raspberrypi.org/debian/ buster main ui' | sudo tee /etc/apt/sources.list.d/raspi.list";
+
+RUN_ONCE "sudo rm /etc/apt/sources.list.d/raspi";

jobs/avahi-daemon/avahi-daemon.job

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt install avahi-daemon"

jobs/bin-folder/bin-folder.job

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-get update";
+RUN "sudo apt-get install git";
+RUN "git clone https://gitlab.com/sbrl/bin.git"
+RUN "cd \${HOME}/bin && GPG_SIGN_COMMITS=n ./envsetup";

jobs/collectd/collectd.conf

@@ -0,0 +1,59 @@
+Interval	300
+
+LoadPlugin	load
+LoadPlugin	thermal
+LoadPlugin	memory
+LoadPlugin	disk
+LoadPlugin	swap
+LoadPlugin	uptime
+#LoadPlugin	smart
+
+LoadPlugin	df
+LoadPlugin	irq
+LoadPlugin	contextswitch
+LoadPlugin	interface
+LoadPlugin	ping
+#LoadPlugin	nginx
+#LoadPlugin	ntpd
+
+LoadPlugin	exec
+
+LoadPlugin	processes
+
+LoadPlugin	network
+
+<Plugin df>
+	MountPoint	"/"
+	MountPoint	"/boot"
+</Plugin>
+
+<Plugin interface>
+	Interface	"eth0"
+	IgnoreSelected	false
+</Plugin>
+
+<Plugin ping>
+	Host "elessar.mooncarrot.space"
+	Host "wopplefox.mooncarrot.space"
+	Host "ubuntu.mirrors.ovh.net"
+	#Host "s3-eu-west-1.amazonaws.com"
+	#Host "github.com"
+	#Host "api.backblazeb2.com"
+</Plugin>
+
+#<Plugin ntpd>
+#	Host	"localhost"
+#	Port	"123"
+#</Plugin>
+
+<Plugin processes>
+	CollectMemoryMaps	false
+</Plugin>
+
+<Plugin "network">
+	<Server "5.198.44.45">
+		SecurityLevel	"Encrypt"
+		Username		"cluster"
+		Password		"{{{PASSWORD}}}"
+	</Server>
+</Plugin>

jobs/collectd/collectd.job

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+if [[ -z "${COLLECTD_PW}" ]]; then
+	echo "Error: COLLECTD_PW environment variable is not set";
+	exit 1;
+fi
+
+RUN "sudo apt-get install -y collectd liboping0";
+
+tmpfile_collectd="$(mktemp --tmpdir "collectd-XXXXXXX")";
+
+chmod 0600 "${tmpfile_collectd}";
+sed -e "s/{{{PASSWORD}}}/${COLLECTD_PW}/g" <"${JOBFILE_DIR}/collectd.conf" >"${tmpfile_collectd}";
+unset COLLECTD_PW;
+
+COPY "${tmpfile_collectd}" "/tmp/collectd.conf";
+
+rm "${tmpfile_collectd}";
+
+RUN "sudo chown root:root /tmp/collectd.conf";
+RUN "sudo chmod 0600 /tmp/collectd.conf";
+RUN "sudo mv /tmp/collectd.conf /etc/collectd/collectd.conf";
+
+RUN "sudo systemctl restart collectd.service";
+RUN "sudo systemctl enable collectd.service";

jobs/config-git/config-git.job

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-get install --yes git openssh-client";
+
+RUN 'ssh -T git@git.starbeamrainbowlabs.com || ssh-keyscan -H git.starbeamrainbowlabs.com >>$HOME/.ssh/known_hosts';
+RUN 'sudo -E ssh -T git@git.starbeamrainbowlabs.com || ssh-keyscan -H git.starbeamrainbowlabs.com | sudo tee -a /root/.ssh/known_hosts';
+
+RUN "[[ -d /etc/cluster-config ]] && { cd /etc/cluster-config; sudo -E git pull; } || sudo -E git clone git@git.starbeamrainbowlabs.com:sbrl/cluster-config.git /etc/cluster-config";

jobs/consul-client/consul-client.job

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+SCRIPT "jobs/consul-server/common.sh";
+
+COPY "../consul/client.hcl" "/tmp/client.hcl"
+
+RUN "sudo mv /tmp/client.hcl /etc/consul/client.hcl";
+RUN "sudo chown root:root /etc/consul/client.hcl";
+RUN "sudo apt-get update";
+RUN "sudo apt-get install --yes hashicorp-consul-systemd-client";
+
+# Do we need these as a client? I'm guessing not....?
+# If we run into issues, we'll unblock them.
+# RUN "sudo ufw allow 8301 comment consul-serf-lan";
+# # RUN "sudo ufw allow 8300/tcp comment consul-rpc";
+# RUN "sudo ufw allow 8600 comment consul-dns";
+
+# Commented out since we're now running over wgoverlay & allowing all traffic on that interface because dynamic ports
+# Delete old ufw rules
+RUN "sudo ufw delete allow 8301 comment consul-serf-lan";
+RUN "sudo ufw delete allow 8300/tcp comment consul-rpc";
+RUN "sudo ufw delete allow 8600 comment consul-dns";
+
+RUN "sudo systemctl enable consul.service";
+RUN "sudo systemctl restart consul.service";

jobs/consul-server/common.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+log_msg() {
+	echo -e "[ ${HOSTNAME} ] [ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $@" >&2;
+}
+
+
+log_msg "Installing consul";
+sudo apt-get update;
+sudo apt-get install --yes hashicorp-consul;
+
+log_msg "consul installed successfully. Version:";
+consul --version;
+
+log_msg "Creating directories";
+if [[ ! -d "/etc/consul" ]]; then
+	sudo mkdir /etc/consul;
+	sudo chown root:root /etc/consul;
+fi
+
+if [[ ! -d "/srv/consul" ]]; then
+	sudo mkdir /srv/consul;
+	sudo chown root:root /srv/consul;
+	sudo chmod 0750 /srv/consul;
+fi

jobs/consul-server/consul-server.job

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+SCRIPT "${JOBFILE_DIR}/common.sh";
+
+COPY "../consul/server.hcl" "/tmp/server.hcl"
+
+RUN "sudo mv /tmp/server.hcl /etc/consul/server.hcl";
+RUN "sudo chown root:root /etc/consul/server.hcl";
+RUN "sudo apt-get update";
+RUN "sudo apt-get install --yes hashicorp-consul-systemd-server";
+
+# TODO: Restrict these to the wesher subnet, and add 8500/tcp (the HTTP API)
+# TODO: Do that for Nomad too
+# RUN "sudo ufw allow 8301 comment consul-serf-lan";
+# RUN "sudo ufw allow 8300/tcp comment consul-rpc";
+# RUN "sudo ufw allow 8600 comment consul-dns";
+
+# Commented out since we're now running over wgoverlay & allowing all traffic on that interface because dynamic ports
+# Delete old ufw rules
+RUN "sudo ufw delete allow 8301 comment consul-serf-lan";
+RUN "sudo ufw delete allow 8300/tcp comment consul-rpc";
+RUN "sudo ufw delete allow 8600 comment consul-dns";
+
+RUN "sudo systemctl enable consul.service";
+RUN "sudo systemctl restart consul.service";

jobs/dns/dns.job

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+SCRIPT "${JOBFILE_DIR}/dns.sh";

jobs/dns/dns.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Usage:
+#	configLine [searchPattern] [replaceLine] [filePath]
+# Ref https://stackoverflow.com/a/54909102/1460422
+config_line() {
+	local OLD_LINE_PATTERN=$1; shift
+	local NEW_LINE=$1; shift
+	local FILE=$1
+	local NEW=$(echo "${NEW_LINE}" | sed 's/\//\\\//g')
+	touch "${FILE}"
+	sed -i '/'"${OLD_LINE_PATTERN}"'/{s/.*/'"${NEW}"'/;h};${x;/./{x;q100};x}' "${FILE}"
+	if [[ $? -ne 100 ]] && [[ ${NEW_LINE} != '' ]]
+	then
+		echo "${NEW_LINE}" >> "${FILE}"
+	fi
+}
+
+log_msg() {
+	echo -e "[ ${HOSTNAME} ] [ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $@" >&2;
+}
+
+# Old approach - we've learned we can override /etc/network/interfaces via /etc/dhcp/dhclient.conf
+# config_line "dns-nameservers" "dns-nameservers 172.16.230.100" "/etc/network/interfaces";
+
+completed_file="/etc/dhcp/sbrl-dns-configured";
+
+log_msg "Configuring DNS nameserver";
+if [[ ! -f "${completed_file}" ]]; then
+	log_msg "DNS nameserver not yet configured, appending to /etc/dhcp/dhclient.conf";
+	echo "interface \"eth0\" {
+	supersede domain-name-servers 172.16.230.100;
+}" | sudo tee -a /etc/dhcp/dhclient.conf;
+	sudo touch "${completed_file}";
+	
+	log_msg "Complete, this machine needs a reboot to activate the new config directives";
+else
+	log_msg "DNS nameserver config written already, no changes made";
+fi

jobs/docker/docker.job

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Fix apt update errors
+# Ref https://blog.samcater.com/fix-workaround-rpi4-docker-libseccomp2-docker-20/
+RUN "sudo apt-get install libseccomp2 -t buster-backports";
+
+# RUN "curl -fsSL https://get.docker.com -o /tmp/get-docker.sh";
+# RUN "chmod +x /tmp/get-docker.sh";
+# RUN "sudo sh /tmp/get-docker.sh";
+RUN 'echo "deb [arch=$(dpkg --print-architecture)] http://download.docker.com/linux/raspbian buster stable" | sudo tee /etc/apt/sources.list.d/docker.list';
+RUN "sudo apt-get update";
+RUN "sudo apt-get install --yes docker-ce";
+RUN "if [[ ! -e \"/etc/docker/daemon.json\" ]]; then echo '{\"insecure-registries\":[\"registry.service.mooncarrot.space:5000\"]}' | sudo tee /etc/docker/daemon.json; fi";
+RUN "sudo systemctl reload docker.service";

jobs/install-utils-python/install-utils-python.job

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-get install python3-pip";
+
+RUN "sudo pip3 install setuptools";
+RUN "sudo pip3 install quarry";

jobs/install-utils/install-utils.job

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-get install --no-install-recommends --yes git net-tools dnsutils uptimed software-properties-common iftop iotop pv screen netcat f3 less optipng jpegoptim lsof strace ddrescue ffmpeg lnav jq";

jobs/laminar-sshkey/laminar-sshkey.job

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+sshkey="$(mktemp --tmpdir "laminarci-sshkey-pub-XXXXXXX")";
+echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICzYSWsGbWHh+cDheHJSKlCrxYDLwS6SKwVMV9SAaGsO laminar-ci@elessar" >"${sshkey}";
+
+upload_target="/tmp/laminarci-sshkey-pub";
+
+COPY "${sshkey}" "${upload_target}";
+
+RUN "sudo mkdir -p /root/.ssh";
+RUN_ONCE "cat '${upload_target}' | sudo tee /root/.ssh/authorized_keys";
+RUN "rm '${upload_target}'";
+
+# Set permissions
+RUN "sudo chown -R root:root /root/.ssh";
+RUN "sudo chmod 0600 /root/.ssh/authorized_keys";

jobs/maintenance/maintenance.job

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Remove any unused Docker images
+RUN "sudo docker image prune --force";

jobs/nfs/cluster-shared-nfs

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+if ! mountpoint -q /mnt/shared; then mount /mnt/shared; fi
+if ! mountpoint -q /mnt/elfstone; then mount /mnt/elfstone; fi
+if ! mountpoint -q /mnt/elessar-music; then mount /mnt/elessar-music; fi
+if ! mountpoint -q /mnt/elessar-syncthing; then mount /mnt/elessar-syncthing; fi

jobs/nfs/nfs-setup.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+echo "[job/nfs] Checking NFS mounts" >&2;
+
+# _netdev	The filesystem resides on a device that requires network access (used to prevent the system from attempting to mount these filesystems until the network has been enabled on the system)
+# nofail	Do not report errors for this device if it does not exist.
+# auto		Auto-mount on boot
+# noatime	No last-access time
+# tcp		Use tcp, not udp
+# bg		The system won't block during boot until it is able to mount the filesystem, but won't give up on trying to mount it either.
+# timeo=VALUE	The time in deciseconds (tenths of  a  second)  the  NFS client waits for a response before it retries an NFS request.
+# retrans=n		The number of times the NFS client retries a request before it attempts further recovery action.
+
+
+if [[ ! -d /mnt/shared ]]; then
+	echo "[job/nfs] Creating mount for shared cluster data" >&2;
+	sudo mkdir -p /mnt/shared;
+	echo 'magicbag.node.mooncarrot.space:/mnt/elfstone2/cluster	/mnt/shared	nfs	auto,nofail,noatime,_netdev,tcp,bg,timeo=50,retrans=5	0	0' | sudo tee -a /etc/fstab;
+	sudo mount /mnt/shared;
+fi
+
+
+if [[ ! -d /mnt/elfstone ]]; then
+	echo "[job/nfs] Creating mount for elfstone" >&2;
+	sudo mkdir -p /mnt/elfstone;
+	echo 'magicbag.node.mooncarrot.space:/mnt/elfstone2/main	/mnt/elfstone	nfs	auto,nofail,noatime,_netdev,tcp,bg,timeo=50,retrans=5	0	0' | sudo tee -a /etc/fstab;
+	sudo mount /mnt/elfstone;
+fi
+
+
+if [[ ! -d /mnt/elessar-music ]]; then
+	echo "[job/nfs] Creating mount for elessar-music" >&2;
+	sudo mkdir -p /mnt/elessar-music;
+	echo 'magicbag.node.mooncarrot.space:/mnt/elfstone2/syncthing/Music	/mnt/elessar-music	nfs	auto,nofail,_netdev,noatime,tcp,bg,timeo=50,retrans=5	0	0' | sudo tee -a /etc/fstab;
+	sudo mount /mnt/elessar-music;
+fi
+
+
+if [[ ! -d /mnt/elessar-syncthing ]]; then
+	echo "[job/nfs] Creating mount for elessar-syncthing" >&2;
+	sudo mkdir -p /mnt/elessar-syncthing;
+	echo 'magicbag.node.mooncarrot.space:/mnt/elfstone2/syncthing	/mnt/elessar-syncthing	nfs	auto,nofail,_netdev,noatime,tcp,bg,timeo=50,retrans=5	0	0' | sudo tee -a /etc/fstab;
+	sudo mount /mnt/elessar-syncthing;
+fi
+
+echo "[job/nfs] Complete" >&2;

jobs/nfs/nfs.job

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-get update";
+RUN "sudo apt-get install --yes nfs-common";
+
+SCRIPT "${JOBFILE_DIR}/nfs-setup.sh"
+
+COPY "${JOBFILE_DIR}/cluster-shared-nfs" "/tmp/cluster-shared-nfs";
+RUN "sudo mv /tmp/cluster-shared-nfs /etc/network/if-up.d/cluster-shared-nfs";
+RUN "sudo chown root:root /etc/network/if-up.d/cluster-shared-nfs";
+RUN "sudo chmod +x /etc/network/if-up.d/cluster-shared-nfs"

jobs/nomad-both/common.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+log_msg() {
+	echo -e "[ ${HOSTNAME} ] [ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $*" >&2;
+}
+
+
+log_msg "Installing nomad";
+sudo apt-get update;
+sudo apt-get install --yes hashicorp-nomad;
+
+log_msg "nomad installed successfully. Version:";
+nomad --version;
+
+log_msg "Creating directories";
+if [[ ! -d "/etc/nomad" ]]; then
+	sudo mkdir /etc/nomad;
+	sudo chown root:root /etc/nomad;
+fi
+
+if [[ ! -d "/srv/nomad" ]]; then
+	sudo mkdir /srv/nomad;
+	sudo chown root:root /srv/nomad;
+	sudo chmod 0750 /srv/nomad;
+fi
+
+check_cgroups_memory() {
+	log_msg "Checking memory cgroups";
+	
+	cgroups_enabled="$(awk '/memory/ { print $2 }' < /proc/cgroups)";
+	
+	if [[ "${cgroups_enabled}" -ne 0 ]]; then
+		log_msg "memory cgroups already enabled";
+		return 0;
+	fi
+	
+	
+	filepath_cmdline="/boot/cmdline.txt";
+	if [[ ! -e "${filepath_cmdline}" ]]; then
+		filepath_cmdline="/boot/firmware/cmdline.txt";
+	fi
+	if [[ ! -e "${filepath_cmdline}" ]]; then
+		log_msg "Failed to find cmdline.txt; can't check for cgroups";
+		return 1;
+	fi
+	
+	if grep -q cgroup_enable=memory /boot/cmdline.txt; then
+		log_msg "memory cgroups already present in cmdline.txt, a reboot is required to apply the update";
+		return 0;
+	fi
+	
+	log_msg "memory cgroups not present in cmdline.txt, enabling....";
+	(tr -d '\n' <"${filepath_cmdline}" && echo " cgroup_enable=memory cgroup_memory=1") | sudo tee "${filepath_cmdline}.new";
+	
+	sudo mv "${filepath_cmdline}" "${filepath_cmdline}.old-$(date +"%Y-%m-%d")";
+	sudo mv "${filepath_cmdline}.new" "${filepath_cmdline}";
+	
+	log_msg "New contents of cmdline.txt:";
+	cat "${filepath_cmdline}";
+	log_msg "A reboot is required to apply the changes.";
+}
+
+check_cgroups_memory;

jobs/nomad-both/nomad-both.job

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+SCRIPT "${JOBFILE_DIR}/common.sh";
+
+COPY "../nomad/both.hcl" "/tmp/both.hcl"
+
+RUN "sudo mv /tmp/both.hcl /etc/nomad/both.hcl";
+RUN "sudo chown root:root /etc/nomad/both.hcl";
+
+RUN "sudo apt-get install --yes hashicorp-nomad-systemd-both";
+
+
+# Commented out since we're now running over wgoverlay & allowing all traffic on that interface because dynamic ports
+# RUN "sudo ufw allow 4646/tcp comment nomad-http";
+# RUN "sudo ufw allow 4647/tcp comment nomad-rpc";
+# RUN "sudo ufw allow 4648/tcp comment nomad-serf";
+RUN "sudo ufw delete allow 4646/tcp comment nomad-http";
+RUN "sudo ufw delete allow 4647/tcp comment nomad-rpc";
+RUN "sudo ufw delete allow 4648/tcp comment nomad-serf";
+
+RUN "sudo systemctl enable nomad.service";
+RUN "sudo systemctl restart nomad.service";

jobs/nomad-client/nomad-client.job

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+SCRIPT "jobs/nomad-both/common.sh";
+
+COPY "../nomad/client.hcl" "/tmp/client.hcl"
+
+RUN "sudo mv /tmp/client.hcl /etc/nomad/client.hcl";
+RUN "sudo chown root:root /etc/nomad/client.hcl";
+
+RUN "sudo apt-get install --yes hashicorp-nomad-systemd-client";
+
+# Commented out since we're now running over wgoverlay & allowing all traffic on that interface because dynamic ports
+# RUN "sudo ufw allow 4646/tcp comment nomad-http";
+# RUN "sudo ufw allow 4647/tcp comment nomad-rpc";
+# RUN "sudo ufw allow 4648/tcp comment nomad-serf";
+RUN "sudo ufw delete allow 4646/tcp comment nomad-http";
+RUN "sudo ufw delete allow 4647/tcp comment nomad-rpc";
+RUN "sudo ufw delete allow 4648/tcp comment nomad-serf";
+
+RUN "sudo systemctl enable nomad.service";
+RUN "sudo systemctl restart nomad.service";

jobs/preconfigure/preconfigure.job

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-get install -y logrotate"
+
+COPY "${JOBFILE_DIR}/misc" "/tmp/misc"
+RUN "sudo mv /tmp/misc /etc/logrotate.d/misc"
+RUN "sudo chown root:root /etc/logrotate.d/misc"
+
+# Fix logrotate issue
+RUN "sudo chmod 0750 /var/log"
+RUN "sudo systemctl restart logrotate"

jobs/sbrl-apt-repo/sbrl-apt-repo.job

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+RUN "echo 'deb http://apt.starbeamrainbowlabs.com/ ./ # apt.starbeamrainbowlabs.com' | sudo tee /etc/apt/sources.list.d/sbrl.list";
+
+RUN "sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D48D801C6A66A5D8";
+
+RUN "sudo apt-get update";

jobs/ssh-preconnect/ssh-preconnect.job

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+RUN 'ssh -T git@git.starbeamrainbowlabs.com || ssh-keyscan -H git.starbeamrainbowlabs.com >>$HOME/.ssh/known_hosts';
+RUN 'sudo -E ssh -T git@git.starbeamrainbowlabs.com || ssh-keyscan -H git.starbeamrainbowlabs.com | sudo tee -a /root/.ssh/known_hosts';

jobs/test/test.job

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+RUN "uptime";
+
+COPY "${JOBFILE_DIR}/testfile.txt" "/tmp";
+
+RUN "rm /tmp/testfile.txt";

jobs/test/testfile.txt

@@ -0,0 +1 @@
+it works!

jobs/ufw/ufw.job

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-get update";
+
+RUN "sudo apt-get install --yes ufw";
+
+RUN "sudo ufw allow 22/tcp comment ssh";
+
+RUN "echo 'y' | sudo ufw enable";

jobs/update-system/update-system.job

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+SCRIPT "${JOBFILE_DIR}/update-system.sh";

jobs/update-system/update-system.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+log_msg() {
+	echo -e "[ ${HOSTNAME} ] [ $(date -u +"%Y-%m-%dT%H:%M:%SZ") ] $@" >&2;
+}
+
+log_msg "Starting update";
+sudo apt-get update;
+
+log_msg "Starting upgrade";
+sudo apt-get dist-upgrade --yes;
+
+log_msg "Starting autoremove";
+sudo apt-get autoremove --yes;
+
+log_msg "Starting autoclean";
+sudo apt-get autoclean;

jobs/vault/vault.job

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-get install -y hashicorp-vault hashicorp-vault-systemd";
+
+COPY "../vault/server.hcl" "/tmp/vault-server.hcl";
+RUN "sudo chown root:root /tmp/vault-server.hcl";
+RUN "sudo mkdir -p /etc/vault";
+RUN "sudo mv /tmp/vault-server.hcl /etc/vault/server.hcl";
+
+RUN "sudo systemctl enable vault.service";
+RUN "sudo systemctl start vault.service";

jobs/wesher/wesher.job

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+echo "Don't forget to run the wireguard job first";
+
+# Sort out log rotation
+COPY "${JOBFILE_DIR}/wesher.logrotate" "/tmp/wesher.logrotate"
+RUN "sudo mv /tmp/wesher.logrotate /etc/logrotate.d/wesher"
+RUN "sudo chown root:root /etc/logrotate.d/wesher"
+
+RUN "sudo apt-get update";
+
+RUN "sudo apt-get install --yes wesher wesher-systemd";
+
+RUN "sudo ufw allow 7946 comment wesher-gossip";
+RUN "sudo ufw allow 51820/udp comment wesher-wireguard";
+
+# Allow all traffic from the VPN; we don't actually know what ports Nomad will use
+RUN "sudo ufw allow in on wgoverlay";
+
+# Activate the new logging system
+RUN "sudo systemctl restart logrotate"
+
+# See the Keepass entry in our database for copy-paste commands to join nodes.
+# Basically:
+#	sudo wesher --cluster-key CLUSTER_KEY_HERE --join 172.16.230.100 --overlay-net 172.31.250.0/16 --log-level info
+#	sudo systemctl enable --now wesher.service
+#	sudo systemctl status wesher.service

jobs/wesher/wesher.logrotate

@@ -0,0 +1,12 @@
+/var/log/wesher/*.log {
+   rotate 12
+   weekly
+   missingok
+   notifempty
+   compress
+   delaycompress
+   sharedscripts
+   postrotate
+	   invoke-rc.d rsyslog rotate >/dev/null 2>&1 || true
+   endscript
+}

jobs/wireguard/wireguard.job

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+RUN "sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 04EE7237B7D453EC 648ACFD622F3D138";
+
+RUN "echo 'deb http://httpredir.debian.org/debian buster-backports main contrib non-free' | sudo tee /etc/apt/sources.list.d/debian-backports.list";
+
+RUN "sudo apt-get update";
+
+RUN "sudo apt-get install --yes raspberrypi-kernel-headers";
+
+RUN "sudo apt-get install --yes wireguard";

list-hostgroups

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Make sure the current directory is the location of this script to simplify matters
+cd "$(dirname "$(readlink -f "$0")")";
+
+find hosts -type f -iname '*.txt' | xargs -n1 basename --suffix=.txt | sort

list-jobs

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Make sure the current directory is the location of this script to simplify matters
+cd "$(dirname "$(readlink -f "$0")")";
+
+ls ./jobs/

new-job

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Make sure the current directory is the location of this script to simplify matters
+cd "$(dirname "$(readlink -f "$0")")";
+
+job_name="$1";
+
+if [[ -z "${job_name}" ]]; then
+	echo "Usage:";
+	echo "	./new-job {job_name}";
+	exit;
+fi
+
+job_filename="./jobs/${job_name}/${job_name}.job";
+
+mkdir -p "$(dirname "${job_filename}")";
+
+echo -e "#!/usr/bin/env bash\n" >"${job_filename}";
+chmod +x "${job_filename}";

ssh-config

@@ -0,0 +1,46 @@
+ServerAliveInterval		120
+
+Host *
+	ControlMaster auto
+	ControlPath /tmp/%r@%h:%p
+
+
+Host cluster-wopplefox
+	Hostname 172.16.230.100
+	Port 22
+	
+	User root
+	IdentityFile ~/.ssh/id_ed25519
+	ForwardAgent yes
+Host cluster-sycadil
+	Hostname 172.16.230.102
+	Port 22
+	
+	User dietpi
+	IdentityFile ~/.ssh/id_ed25519
+	ForwardAgent yes
+Host cluster-spatterling
+	Hostname 172.16.230.101
+	Port 22
+	
+	User dietpi
+	IdentityFile ~/.ssh/id_ed25519
+	ForwardAgent yes
+Host cluster-balusha
+	Hostname 172.16.230.103
+	Port 22
+	
+	User dietpi
+	IdentityFile ~/.ssh/id_ed25519
+	ForwardAgent yes
+Host cluster-silverleaf
+	# braskin
+	# toko
+	# *silverleaf*
+	# elligon: router
+	Hostname 172.16.230.104
+	Port 22
+	
+	User dietpi
+	IdentityFile ~/.ssh/id_ed25519
+	ForwardAgent yes