Import from main cluster config repo.

Ref https://git.starbeamrainbowlabs.com/sbrl/cluster-config

.gitignore

@@ -0,0 +1,59 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/git
+# Edit at https://www.toptal.com/developers/gitignore?templates=git
+
+### Git ###
+# Created by git for backups. To disable backups in Git:
+# $ git config --global mergetool.keepBackup false
+*.orig
+
+# Created by git when using merge tools for conflicts
+*.BACKUP.*
+*.BASE.*
+*.LOCAL.*
+*.REMOTE.*
+*_BACKUP_*.txt
+*_BASE_*.txt
+*_LOCAL_*.txt
+*_REMOTE_*.txt
+
+# End of https://www.toptal.com/developers/gitignore/api/git
+
+# Created by https://www.toptal.com/developers/gitignore/api/archives
+# Edit at https://www.toptal.com/developers/gitignore?templates=archives
+
+### Archives ###
+# It's better to unpack these files and commit the raw source because
+# git has its own built in compression methods.
+*.7z
+*.jar
+*.rar
+*.zip
+*.gz
+*.gzip
+*.tgz
+*.bzip
+*.bzip2
+*.bz2
+*.xz
+*.lzma
+*.cab
+*.xar
+
+# Packing-only formats
+*.iso
+*.tar
+
+# Package management formats
+*.dmg
+*.xpi
+*.gem
+*.egg
+*.deb
+*.rpm
+*.msi
+*.msm
+*.msp
+*.txz
+
+# End of https://www.toptal.com/developers/gitignore/api/archives

imagebuilder.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+
+if [[ ! -f "/.dockerenv" ]]; then
+	repo_root="$(git rev-parse --show-toplevel)";
+else
+	repo_root="/srv";
+fi
+
+lantern_path="${repo_root}/lantern-build-engine";
+
+IMAGEBUILDER_REGISTRY="${IMAGEBUILDER_REGISTRY:-registry.service.mooncarrot.space:5000}";
+
+###############################################################################
+
+#shellcheck disable=SC1090
+source "${lantern_path}/lantern.sh";
+
+if [[ -z "${BASE_PATH}" ]]; then
+	subtask_begin "BASE_PATH environment variable not found - setting base path to ${HC}${PWD}${RS}";
+	BASE_PATH="${PWD}";
+	subtask_end "$?";
+fi
+if [[ ! -d "${BASE_PATH}" ]]; then
+	echo "Error: The specified base path '${BASE_PATH}' doesn't exist.";
+	exit 1;
+fi
+
+# Make sure the current directory is the location of this script to simplify matters
+cd "$(dirname "$(readlink -f "$0")")" || { echo "Error: Failed to cd"; exit 1; };
+
+# Check out the lantern git submodule if needed
+if [ ! -f "${lantern_path}/lantern.sh" ]; then git submodule update --init "${lantern_path}"; fi
+
+# Create temporary directory
+temp_dir="$(mktemp --tmpdir -d "imagebuilder-XXXXXXX")";
+on_exit() {
+	task_begin "Cleaning up temporary directory";
+	rm -rf "${temp_dir}";
+	task_end "$?";
+}
+trap on_exit EXIT;
+
+
+###############################################################################
+
+subcommand="${1}";
+shift;
+
+if [[ -z "${subcommand}" ]]; then
+	CHEADING="${HC}${FCYN}";
+	CACTION="${FYEL}";#
+	CARG="${FMAG}"
+	echo -e "${HC}imagebuilder: Docker image (re)builder${RS}" >&2;
+	echo -e "	By Starbeamrainbowlabs" >&2;
+	echo -e "" >&2;
+	echo -e "${CHEADING}Usage:${RS}" >&2;
+	echo -e "	${HC}${FGRN}./imagebuilder.sh${RS} ${CACTION}{action}${RS} ${LC}[${RS}${CARG}{arguments}${RS}${LC}]${RS}" >&2;
+	echo -e "" >&2;
+	echo -e "${CHEADING}Actions:${RS}" >&2;
+	echo -e "	${CACTION}build${RS} ${CARG}{imagename}${RS}" >&2;
+	echo -e "		Build the given image and upload it to the docker registry" >&2;
+	echo -e "	${CACTION}list${RS}" >&2;
+	echo -e "		List available images" >&2;
+	echo -e "" >&2;
+	echo -e "${CHEADING}Environment Variables:${RS}" >&2;
+	echo -e "	${CACTION}IMAGEBUILDER_REGISTRY${RS}" >&2;
+	echo -e "		Set the url of the Docker registry (default: ${HC}registry.service.mooncarrot.space:5000${RS})" >&2;
+	echo -e "	${CACTION}BASE_PATH${RS}" >&2;
+	echo -e "		Base path in which to look for image directories (defaults to the current working directory)" >&2;
+	echo -e "" >&2;
+	exit 0;
+fi
+
+###############################################################################
+
+case "${subcommand}" in
+	list)
+		while read -r filepath; do
+			filepath_stripped="${filepath#"${BASE_PATH}"}";
+			if [[ -z "${filepath_stripped}" ]] || [[ ! -f "${filepath}/type.txt" ]]; then
+				continue;
+			fi
+			
+			echo "${filepath_stripped}";
+		done < <(find "${BASE_PATH}" -maxdepth 1 -type d);
+		;;
+	
+	build)
+		imagename="${1}";
+		
+		if [[ -z "${imagename}" ]]; then
+			echo "Error: No image name specified.":
+			exit 1;
+		fi
+		
+		imagedir="${BASE_PATH}/${imagename}";
+		
+		if [[ ! -d "${imagedir}" ]]; then
+			echo -e "Error: An image with the name ${HC}${imagename}${RS} doesn't exist.";
+			exit 2;
+		fi
+		
+		if [[ ! -f "${imagedir}/type.txt" ]]; then
+			echo -e "Error: No type.txt file was found for the image with the name ${HC}${imagename}${RS}.";
+			exit 3;
+		fi
+		
+		imagetype="$(tr -d "[:blank:]" <"${imagedir}/type.txt")";
+		
+		case "${imagetype}" in
+			docker)
+				if [[ ! -f "${imagedir}/Dockerfile" ]]; then
+					echo -e "Error: Failed to find a Dockerfile at ${HC}${imagedir}/Dockerfile${RS}.";
+					exit 7;
+				fi
+				
+				cd "${imagedir}" || { echo -e "Error: Failed to cd into ${HC}${imagedir}${RS}"; exit 1; };
+				
+				if [[ -x "./pre.sh" ]]; then
+					task_begin "Executing pre-build hook";
+					execute ./pre.sh;
+					task_end "$?";
+				fi
+				
+				docker_tag="${IMAGEBUILDER_REGISTRY}/${imagename}";
+				task_begin "Building docker image";
+				echo "Tag: ${docker_tag}";
+				execute docker build --no-cache --pull --tag "${docker_tag}" --build-arg "REPO_LOCATION=${IMAGEBUILDER_REGISTRY}/" .;
+				task_end "$?";
+				
+				task_begin "Pushing resulting docker image";
+				execute docker push "${docker_tag}";
+				task_end "$?";
+				
+				if [[ -x "./post.sh" ]]; then
+					task_begin "Executing post-build hook";
+					execute ./post.sh;
+					task_end "$?";
+				fi
+				;;
+			
+			base|base-nopush)
+				builderscript="${imagedir}/${imagename}.sh";
+				if [[ ! -x "${builderscript}" ]]; then
+					echo -e "Error: Failed to find the base image builder script at ${HC}${builderscript}${RS} (is it executable?).";
+					exit 5;
+				fi
+				
+				output_dir="${temp_dir}/${imagename}";
+				
+				task_begin "Building base image";
+				if [[ "${UID}" -ne 0 ]] && which fakeroot && which fakechroot; then
+					echo "Non-root user detected - using fakeroot & fakechroot";
+					execute fakechroot fakeroot "${builderscript}" "${output_dir}";
+				else
+					echo "root user or fakeroot & fakechroot not detected";
+					execute "${builderscript}" "${output_dir}";
+				fi
+				task_end "$?";
+				
+				if [[ "${subcommand}" == "base-nopush" ]]; then
+					echo "Nopush mode invoked, not checking output directory or pushing to docker registry";
+					exit 0;
+				fi
+				
+				if [[ ! -d "${output_dir}" ]]; then
+					echo -e "Error: The builder script failed to create the output directory.";
+					exit 6;
+				fi
+				
+				task_begin "Importing resulting image into Docker";
+				image_filepath="$(find "${output_dir}" -iname "*.tar.gz" -printf '%p' -quit)";
+				docker_tag="$(docker import - <"${image_filepath}")";
+				task_end "$?";
+				
+				task_begin "Tagging and pushing to registry";
+				docker_tag_push="${IMAGEBUILDER_REGISTRY}/${imagename}"
+				execute docker tag "${docker_tag}" "${docker_tag_push}";
+				execute docker push "${docker_tag_push}";
+				task_end "$?";
+				;;
+			
+			*)
+				echo -e "Error: The image type ${HC}${imagetype}${RS} was not recognised. Currently recognised types: base, base-nopush, docker";
+				exit 4;
+				;;
+			esac
+		;;
+	
+	*)
+		echo -e "Unknown subcommand '${HC}${subcommand}${RS}' (try calling imagebuilder.sh without any arguments)";
+		;;
+esac

images/docker-registry-ui/Dockerfile

@@ -0,0 +1,24 @@
+ARG REPO_LOCATION
+# ARG BASE_VERSION
+
+FROM ${REPO_LOCATION}minideb-node AS builder
+
+RUN install_packages git ca-certificates
+
+RUN git clone https://github.com/Joxit/docker-registry-ui.git /srv
+WORKDIR /srv
+RUN npm install
+
+###############################################################################
+
+FROM ${REPO_LOCATION}minideb
+# FROM ${REPO_LOCATION}minideb:${BASE_VERSION}
+
+COPY --from=builder /srv/dist /srv
+
+RUN install_packages busybox
+
+USER 3:3
+
+WORKDIR /srv
+ENTRYPOINT /bin/busybox httpd -f -p 5500

images/docker-registry-ui/type.txt

@@ -0,0 +1 @@
+docker

images/imagewrangler/Dockerfile

@@ -0,0 +1,49 @@
+ARG REPO_LOCATION
+
+FROM ${REPO_LOCATION}minideb AS builder
+
+RUN install_packages git curl openssh-client ca-certificates
+
+RUN echo "deb [arch=armhf] http://download.docker.com/linux/debian buster stable" >/etc/apt/sources.list.d/docker.list
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg >/etc/apt/trusted.gpg.d/docker.asc
+
+COPY imagewrangler_ed25519 /tmp/imagewrangler_ed25519
+RUN ssh-keyscan -H git.starbeamrainbowlabs.com >/tmp/known_hosts
+
+# Invalidate the cache to force Docker to pull the latest commit
+ADD datetime.txt /tmp/datetime.txt
+RUN GIT_SSH_COMMAND="ssh -i /tmp/imagewrangler_ed25519 -o PreferredAuthentications=publickey -o UserKnownHostsFile=/tmp/known_hosts" git clone git@git.starbeamrainbowlabs.com:sbrl/cluster-config.git /srv
+WORKDIR /srv
+RUN git submodule update --init
+
+###############################################################################
+
+FROM ${REPO_LOCATION}minideb
+
+# Docker apt repo
+COPY --from=builder /etc/apt/trusted.gpg.d/docker.asc /etc/apt/trusted.gpg.d/docker.asc
+COPY --from=builder /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/docker.list
+
+# Everything from make onwards is needed for minideb
+RUN install_packages curl jq docker-ce-cli ca-certificates fakeroot fakechroot git
+
+# These will probably invalidate the cache, so we install the packages above first
+COPY --from=builder /srv/lantern-build-engine /srv/lantern-build-engine
+COPY --from=builder /srv/docker /srv/docker
+COPY --from=builder /srv/scripts /srv/scripts
+
+# Note that we chown here because COPY --chown is apparently unreliable :-(
+RUN groupadd --gid 995 docker && \
+	useradd --no-create-home --system --uid 50 --groups docker imagewrangler && \
+	chown -R 50:995 /srv/docker && \
+	mkdir /mnt/data_dir
+
+# We need the docker socket to enable us to start containers in order to check them
+VOLUME /run/docker.sock
+VOLUME /mnt/data_dir
+
+# 995 = the docker group on docker.sock
+USER imagewrangler:docker
+
+WORKDIR /srv/scripts
+ENTRYPOINT [ "/bin/bash", "./imagewrangler.sh", "check" ]

images/imagewrangler/post.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+rm -f imagewrangler_ed25519 imagewrangler_ed25519.pub dateetime.txt;

images/imagewrangler/pre.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+cp /mnt/shared/imagewrangler/imagewrangler_ed25519 .
+cp /mnt/shared/imagewrangler/imagewrangler_ed25519.pub .
+
+date >./datetime.txt

images/imagewrangler/type.txt

@@ -0,0 +1 @@
+docker

images/minetest-mapserver/Dockerfile

@@ -0,0 +1,10 @@
+FROM scratch
+
+ADD ./mapserver /
+
+USER 113:60
+WORKDIR /world
+
+# CMD is executed as an argument to ENTRYPOINT.
+# Ref https://stackoverflow.com/a/21564990/1460422
+ENTRYPOINT ["/mapserver"]

images/minetest-mapserver/minetest-mapserver.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# The architecture - possible values: arm (default), x86, x86_64
+arch="${arch:-arm}";
+
+download_url="https://github.com/minetest-mapserver/mapserver/releases/latest/download/mapserver-linux-${arch}";
+
+# The registry to push to
+DOCKER_REGISTRY="${DOCKER_REGISTRY:-registry.service.mooncarrot.space:5000/}";
+if [[ "${DOCKER_REGISTRY}" == "hub" ]]; then DOCKER_REGISTRY=""; fi
+
+###############################################################################
+
+# Make sure the current directory is the location of this script to simplify matters
+cd "$(dirname "$(readlink -f "$0")")" || { echo "[minetest-mapserver] Error: Failed to cd"; exit 1; };
+
+temp_dir="$(mktemp --tmpdir -d "minetest-mapserver-XXXXXXX")";
+on_exit() {
+	log_msg "Cleaning up";
+	rm -rf "${temp_dir}";
+}
+trap on_exit EXIT;
+
+# Ref https://stackoverflow.com/a/911213/1460422
+if [ -t 1 ] || [[ ! -z "${FORCE_COLOUR}" ]]; then
+	###################
+	# From lantern.sh #
+	RS="\033[0m"	# reset
+	HC="\033[1m"	# hicolor
+	LC="\033[2m"    # locolor / dim
+	###################
+fi
+
+log_msg() {
+	echo -e "${HC}>>>${RS} ${LC}[ ${SECONDS} ]${RS} $*" >&2;
+}
+
+###############################################################################
+
+
+log_msg "Downloading mapserver";
+curl -sSL -o "${temp_dir}/mapserver" "${download_url}"
+
+chmod +x "${temp_dir}/mapserver";
+
+log_msg "Determining version";
+version="$(curl -sSL https://api.github.com/repos/minetest-mapserver/mapserver/releases/latest | jq --raw-output .tag_name)";
+
+log_msg "Version is ${version}";
+
+log_msg "Preparing docker build context";
+
+cp ./Dockerfile "${temp_dir}/Dockerfile";
+
+log_msg "Building docker image";
+
+docker_image_name="${DOCKER_REGISTRY}minetest-mapserver:${arch}";
+docker build "${temp_dir}" --tag "${docker_image_name}";
+
+log_msg "Pushing to ${DOCKER_REGISTRY}";
+docker push "${docker_image_name}";

images/minetest-mapserver/type.txt

@@ -0,0 +1 @@
+base-nopush

images/minideb-node/Dockerfile

@@ -0,0 +1,10 @@
+ARG REPO_LOCATION
+# ARG BASE_VERSION
+
+# NOTE: This might note be what's required - if possible we might only need the Node.js binary.
+# Perhaps we could have 2 different images: minideb-node and node-min?
+# ANOTHER NOTE: We probably don't need NPM - even in the minideb-node Docker image, as we should be able to npm install outside the container and import it in?
+FROM ${REPO_LOCATION}minideb
+# FROM ${REPO_LOCATION}minideb:${BASE_VERSION}
+
+RUN install_packages libatomic1 nodejs-sbrl

images/minideb-node/delete_npm

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Call this script to delete npm.
+# This is a self-deleting script, so no need to scrub it afterwards :D
+rm -r /usr/local/lib/node/lib/node_modules/npm
+rm /usr/local/bin/delete_npm;

images/minideb-node/type.txt

@@ -0,0 +1 @@
+docker

images/minideb/minideb-raspbian.patch

@@ -0,0 +1,17 @@
+diff --git a/mkimage b/mkimage
+index cd9d2c6..ea68301 100755
+--- a/mkimage
++++ b/mkimage
+@@ -16,10 +16,10 @@ exec 2> >(tee -ia "$LOGFILE" >&2)
+ 
+ DEBOOTSTRAP_DIR=$(mktemp -d)
+ cp -a /usr/share/debootstrap/* "$DEBOOTSTRAP_DIR"
+-cp -a /usr/share/keyrings/debian-archive-keyring.gpg "$DEBOOTSTRAP_DIR"
++cp -a /usr/share/keyrings/raspbian-archive-keyring.gpg "$DEBOOTSTRAP_DIR"
+ cp -a "${ROOT}/debootstrap/"* "${DEBOOTSTRAP_DIR}/scripts"
+ 
+-KEYRING=$DEBOOTSTRAP_DIR/debian-archive-keyring.gpg
++KEYRING=$DEBOOTSTRAP_DIR/raspbian-archive-keyring.gpg
+ 
+ if [ -f "${ROOT}/keys/${DIST}.gpg" ]; then
+     gpg --no-default-keyring --keyring "$KEYRING" --import "${ROOT}/keys/${DIST}.gpg"

images/minideb/minideb.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+set -e;
+
+###
+# Settings
+###
+# These can all be changed via environment variables.
+
+clone_url="${clone_url:-https://github.com/bitnami/minideb.git}";
+image_version="${image_version:-buster}";
+
+apply_patch_raspbian="${apply_patch_raspbian:-true}";
+
+setup_proxy="${setup_proxy:-true}";
+proxy_address="${proxy_address:-http://172.16.230.100:3142}";
+
+###############################################################################
+
+target_directory="${1}";
+
+# Ref https://stackoverflow.com/a/911213/1460422
+if [ -t 1 ] || [[ ! -z "${FORCE_COLOUR}" ]]; then
+	###################
+	# From lantern.sh #
+	RS="\033[0m"	# reset
+	HC="\033[1m"	# hicolor
+	LC="\033[2m"    # locolor / dim
+	###################
+fi
+
+if [[ -z "${target_directory}" ]]; then
+	echo "Usage:" >&2;
+	echo "    $0 {target_directory}" >&2;
+	echo "" >&2;
+	echo "" >&2;
+	exit 0;
+fi
+
+if [[ "${UID}" -ne "0" ]]; then
+	echo "Error: This script must be run as root (as we need to chroot to complete the setup)" >&2;
+	echo "Additional information: You ran this script as ${USER} with uid ${UID}, but we expected uid 0" >&2;
+	exit 1;
+fi
+
+if [[ ! -d "${target_directory}" ]]; then
+	mkdir -p "${target_directory}";
+fi
+
+target_directory="$(realpath "${target_directory}")";
+
+###############################################################################
+
+run_in_chroot() {
+	if [[ ! -d "${temp_dir_postprocess}" ]] || [[ -z "${temp_dir_postprocess}" ]]; then
+		echo "Error: Target directory '${temp_dir_postprocess}' does not exist, so can't chroot into it." >&2;
+		echo "Additional Information: You tried to run '${*}'" >&2;
+		return 1;
+	fi
+	chroot "${temp_dir_postprocess}" "$@";
+}
+
+log_msg() {
+	echo -e "\n${HC}>>>${RS} ${LC}[ ${SECONDS} ]${RS} $*\n"
+}
+
+###############################################################################
+
+# Make sure the current directory is the location of this script to simplify matters
+cd "$(dirname "$(readlink -f "$0")")";
+
+config_dir="${PWD}";
+temp_dir="$(mktemp --tmpdir -d "minideb-build-XXXXXXX")";
+temp_dir_postprocess="${temp_dir}/minideb-${image_version}";
+temp_dir_build="$(mktemp -d "${config_dir}/minideb-tmp-XXXXXXX")";
+
+on_exit() {
+	rm -rf "${temp_dir}";
+	if [[ -e "${temp_dir_build}" ]]; then rm -rf "${temp_dir_build}"; fi
+}
+trap on_exit EXIT;
+
+###############################################################################
+
+log_msg "Cloning minideb";
+git clone "${clone_url}" "${temp_dir}/minideb";
+
+cd "${temp_dir}/minideb";
+if [[ "${apply_patch_raspbian}" == "true" ]]; then
+	log_msg "Applying Raspbian patch";
+	git apply "${config_dir}/minideb-raspbian.patch";
+fi
+
+log_msg "Building minideb base image";
+# Must be run as root
+time TMPDIR="${temp_dir_build}" make "${image_version}";
+
+cp -r "${temp_dir}/minideb/build" "${temp_dir}/build";
+filename_tar="$(find "${temp_dir}/build" -name '*.tar' -print -quit)";
+
+log_msg "Unpacking resulting tar archive";
+mkdir "${temp_dir_postprocess}";
+tar -xf "${filename_tar}" -C "${temp_dir_postprocess}";
+
+# We're finished with minideb now
+rm -rf "${temp_dir}/minideb";
+
+###############################################################################
+
+if [[ "${setup_proxy}" == "true" ]]; then
+	log_msg "Applying apt proxy settings";
+	echo "Acquire::http { Proxy \"${proxy_address}\"; }" | run_in_chroot tee /etc/apt/apt.conf.d/proxy
+fi
+
+# # Fix /dev/null - ref https://unix.stackexchange.com/a/146639/64687
+# run_in_chroot rm -f /dev/null;
+# run_in_chroot mknod -m 666 /dev/null c 1 3;
+# run_in_chroot mknod -m 666 /dev/zero c 1 5
+# run_in_chroot chown root:root /dev/null /dev/zero
+
+# run_in_chroot install_packages gnupg;
+
+log_msg "Adding aptosaurus GPG key";
+# Add apt.starbeamrainbowlabs.com
+# No need to apt update, because install_packages will do this for us
+# Ref https://github.com/bitnami/minideb/blob/6c039b8/mkimage#L181-L206
+# Also ref https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=851774
+gpg_key_url="https://apt.starbeamrainbowlabs.com/aptosaurus.asc";
+echo "deb http://apt.starbeamrainbowlabs.com/ /" | run_in_chroot tee /etc/apt/sources.list.d/sbrl.list;
+curl -sSL "${gpg_key_url}" | run_in_chroot sh -c 'cat >/etc/apt/trusted.gpg.d/sbrl-aptosaurus.asc';
+
+###############################################################################
+
+# Repack the tar file
+log_msg "Repacking archive";
+cd "${temp_dir_postprocess}" || { echo "Failed to cd to unpacked archive directory"; exit 1; };
+tar -caf "${target_directory}/minideb.tar.gz" .;
+cp "${filename_tar}.log" "${target_directory}/minideb.log";
+cp "${filename_tar}.manifest" "${target_directory}/minideb.manifest";
+
+log_msg "Written result to ${target_directory}:";
+
+ls -htFl "${target_directory}";

images/minideb/type.txt

@@ -0,0 +1 @@
+base