Import from main cluster config repo.

Ref https://git.starbeamrainbowlabs.com/sbrl/cluster-config
This commit is contained in:
Starbeamrainbowlabs 2020-08-24 13:06:23 +01:00
commit 052f2da998
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
17 changed files with 584 additions and 0 deletions

59
.gitignore vendored Normal file
View file

@ -0,0 +1,59 @@
# Created by https://www.toptal.com/developers/gitignore/api/git
# Edit at https://www.toptal.com/developers/gitignore?templates=git
### Git ###
# Created by git for backups. To disable backups in Git:
# $ git config --global mergetool.keepBackup false
*.orig
# Created by git when using merge tools for conflicts
*.BACKUP.*
*.BASE.*
*.LOCAL.*
*.REMOTE.*
*_BACKUP_*.txt
*_BASE_*.txt
*_LOCAL_*.txt
*_REMOTE_*.txt
# End of https://www.toptal.com/developers/gitignore/api/git
# Created by https://www.toptal.com/developers/gitignore/api/archives
# Edit at https://www.toptal.com/developers/gitignore?templates=archives
### Archives ###
# It's better to unpack these files and commit the raw source because
# git has its own built in compression methods.
*.7z
*.jar
*.rar
*.zip
*.gz
*.gzip
*.tgz
*.bzip
*.bzip2
*.bz2
*.xz
*.lzma
*.cab
*.xar
# Packing-only formats
*.iso
*.tar
# Package management formats
*.dmg
*.xpi
*.gem
*.egg
*.deb
*.rpm
*.msi
*.msm
*.msp
*.txz
# End of https://www.toptal.com/developers/gitignore/api/archives

193
imagebuilder.sh Executable file
View file

@ -0,0 +1,193 @@
#!/usr/bin/env bash
if [[ ! -f "/.dockerenv" ]]; then
repo_root="$(git rev-parse --show-toplevel)";
else
repo_root="/srv";
fi
lantern_path="${repo_root}/lantern-build-engine";
IMAGEBUILDER_REGISTRY="${IMAGEBUILDER_REGISTRY:-registry.service.mooncarrot.space:5000}";
###############################################################################
#shellcheck disable=SC1090
source "${lantern_path}/lantern.sh";
if [[ -z "${BASE_PATH}" ]]; then
subtask_begin "BASE_PATH environment variable not found - setting base path to ${HC}${PWD}${RS}";
BASE_PATH="${PWD}";
subtask_end "$?";
fi
if [[ ! -d "${BASE_PATH}" ]]; then
echo "Error: The specified base path '${BASE_PATH}' doesn't exist.";
exit 1;
fi
# Make sure the current directory is the location of this script to simplify matters
cd "$(dirname "$(readlink -f "$0")")" || { echo "Error: Failed to cd"; exit 1; };
# Check out the lantern git submodule if needed
if [ ! -f "${lantern_path}/lantern.sh" ]; then git submodule update --init "${lantern_path}"; fi
# Create temporary directory
temp_dir="$(mktemp --tmpdir -d "imagebuilder-XXXXXXX")";
on_exit() {
task_begin "Cleaning up temporary directory";
rm -rf "${temp_dir}";
task_end "$?";
}
trap on_exit EXIT;
###############################################################################
subcommand="${1}";
shift;
if [[ -z "${subcommand}" ]]; then
CHEADING="${HC}${FCYN}";
CACTION="${FYEL}";#
CARG="${FMAG}"
echo -e "${HC}imagebuilder: Docker image (re)builder${RS}" >&2;
echo -e " By Starbeamrainbowlabs" >&2;
echo -e "" >&2;
echo -e "${CHEADING}Usage:${RS}" >&2;
echo -e " ${HC}${FGRN}./imagebuilder.sh${RS} ${CACTION}{action}${RS} ${LC}[${RS}${CARG}{arguments}${RS}${LC}]${RS}" >&2;
echo -e "" >&2;
echo -e "${CHEADING}Actions:${RS}" >&2;
echo -e " ${CACTION}build${RS} ${CARG}{imagename}${RS}" >&2;
echo -e " Build the given image and upload it to the docker registry" >&2;
echo -e " ${CACTION}list${RS}" >&2;
echo -e " List available images" >&2;
echo -e "" >&2;
echo -e "${CHEADING}Environment Variables:${RS}" >&2;
echo -e " ${CACTION}IMAGEBUILDER_REGISTRY${RS}" >&2;
echo -e " Set the url of the Docker registry (default: ${HC}registry.service.mooncarrot.space:5000${RS})" >&2;
echo -e " ${CACTION}BASE_PATH${RS}" >&2;
echo -e " Base path in which to look for image directories (defaults to the current working directory)" >&2;
echo -e "" >&2;
exit 0;
fi
###############################################################################
case "${subcommand}" in
list)
while read -r filepath; do
filepath_stripped="${filepath#"${BASE_PATH}"}";
if [[ -z "${filepath_stripped}" ]] || [[ ! -f "${filepath}/type.txt" ]]; then
continue;
fi
echo "${filepath_stripped}";
done < <(find "${BASE_PATH}" -maxdepth 1 -type d);
;;
build)
imagename="${1}";
if [[ -z "${imagename}" ]]; then
echo "Error: No image name specified.":
exit 1;
fi
imagedir="${BASE_PATH}/${imagename}";
if [[ ! -d "${imagedir}" ]]; then
echo -e "Error: An image with the name ${HC}${imagename}${RS} doesn't exist.";
exit 2;
fi
if [[ ! -f "${imagedir}/type.txt" ]]; then
echo -e "Error: No type.txt file was found for the image with the name ${HC}${imagename}${RS}.";
exit 3;
fi
imagetype="$(tr -d "[:blank:]" <"${imagedir}/type.txt")";
case "${imagetype}" in
docker)
if [[ ! -f "${imagedir}/Dockerfile" ]]; then
echo -e "Error: Failed to find a Dockerfile at ${HC}${imagedir}/Dockerfile${RS}.";
exit 7;
fi
cd "${imagedir}" || { echo -e "Error: Failed to cd into ${HC}${imagedir}${RS}"; exit 1; };
if [[ -x "./pre.sh" ]]; then
task_begin "Executing pre-build hook";
execute ./pre.sh;
task_end "$?";
fi
docker_tag="${IMAGEBUILDER_REGISTRY}/${imagename}";
task_begin "Building docker image";
echo "Tag: ${docker_tag}";
execute docker build --no-cache --pull --tag "${docker_tag}" --build-arg "REPO_LOCATION=${IMAGEBUILDER_REGISTRY}/" .;
task_end "$?";
task_begin "Pushing resulting docker image";
execute docker push "${docker_tag}";
task_end "$?";
if [[ -x "./post.sh" ]]; then
task_begin "Executing post-build hook";
execute ./post.sh;
task_end "$?";
fi
;;
base|base-nopush)
builderscript="${imagedir}/${imagename}.sh";
if [[ ! -x "${builderscript}" ]]; then
echo -e "Error: Failed to find the base image builder script at ${HC}${builderscript}${RS} (is it executable?).";
exit 5;
fi
output_dir="${temp_dir}/${imagename}";
task_begin "Building base image";
if [[ "${UID}" -ne 0 ]] && which fakeroot && which fakechroot; then
echo "Non-root user detected - using fakeroot & fakechroot";
execute fakechroot fakeroot "${builderscript}" "${output_dir}";
else
echo "root user or fakeroot & fakechroot not detected";
execute "${builderscript}" "${output_dir}";
fi
task_end "$?";
if [[ "${subcommand}" == "base-nopush" ]]; then
echo "Nopush mode invoked, not checking output directory or pushing to docker registry";
exit 0;
fi
if [[ ! -d "${output_dir}" ]]; then
echo -e "Error: The builder script failed to create the output directory.";
exit 6;
fi
task_begin "Importing resulting image into Docker";
image_filepath="$(find "${output_dir}" -iname "*.tar.gz" -printf '%p' -quit)";
docker_tag="$(docker import - <"${image_filepath}")";
task_end "$?";
task_begin "Tagging and pushing to registry";
docker_tag_push="${IMAGEBUILDER_REGISTRY}/${imagename}"
execute docker tag "${docker_tag}" "${docker_tag_push}";
execute docker push "${docker_tag_push}";
task_end "$?";
;;
*)
echo -e "Error: The image type ${HC}${imagetype}${RS} was not recognised. Currently recognised types: base, base-nopush, docker";
exit 4;
;;
esac
;;
*)
echo -e "Unknown subcommand '${HC}${subcommand}${RS}' (try calling imagebuilder.sh without any arguments)";
;;
esac

View file

@ -0,0 +1,24 @@
ARG REPO_LOCATION
# ARG BASE_VERSION
FROM ${REPO_LOCATION}minideb-node AS builder
RUN install_packages git ca-certificates
RUN git clone https://github.com/Joxit/docker-registry-ui.git /srv
WORKDIR /srv
RUN npm install
###############################################################################
FROM ${REPO_LOCATION}minideb
# FROM ${REPO_LOCATION}minideb:${BASE_VERSION}
COPY --from=builder /srv/dist /srv
RUN install_packages busybox
USER 3:3
WORKDIR /srv
ENTRYPOINT /bin/busybox httpd -f -p 5500

View file

@ -0,0 +1 @@
docker

View file

@ -0,0 +1,49 @@
ARG REPO_LOCATION
FROM ${REPO_LOCATION}minideb AS builder
RUN install_packages git curl openssh-client ca-certificates
RUN echo "deb [arch=armhf] http://download.docker.com/linux/debian buster stable" >/etc/apt/sources.list.d/docker.list
RUN curl -fsSL https://download.docker.com/linux/debian/gpg >/etc/apt/trusted.gpg.d/docker.asc
COPY imagewrangler_ed25519 /tmp/imagewrangler_ed25519
RUN ssh-keyscan -H git.starbeamrainbowlabs.com >/tmp/known_hosts
# Invalidate the cache to force Docker to pull the latest commit
ADD datetime.txt /tmp/datetime.txt
RUN GIT_SSH_COMMAND="ssh -i /tmp/imagewrangler_ed25519 -o PreferredAuthentications=publickey -o UserKnownHostsFile=/tmp/known_hosts" git clone git@git.starbeamrainbowlabs.com:sbrl/cluster-config.git /srv
WORKDIR /srv
RUN git submodule update --init
###############################################################################
FROM ${REPO_LOCATION}minideb
# Docker apt repo
COPY --from=builder /etc/apt/trusted.gpg.d/docker.asc /etc/apt/trusted.gpg.d/docker.asc
COPY --from=builder /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/docker.list
# Everything from make onwards is needed for minideb
RUN install_packages curl jq docker-ce-cli ca-certificates fakeroot fakechroot git
# These will probably invalidate the cache, so we install the packages above first
COPY --from=builder /srv/lantern-build-engine /srv/lantern-build-engine
COPY --from=builder /srv/docker /srv/docker
COPY --from=builder /srv/scripts /srv/scripts
# Note that we chown here because COPY --chown is apparently unreliable :-(
RUN groupadd --gid 995 docker && \
useradd --no-create-home --system --uid 50 --groups docker imagewrangler && \
chown -R 50:995 /srv/docker && \
mkdir /mnt/data_dir
# We need the docker socket to enable us to start containers in order to check them
VOLUME /run/docker.sock
VOLUME /mnt/data_dir
# 995 = the docker group on docker.sock
USER imagewrangler:docker
WORKDIR /srv/scripts
ENTRYPOINT [ "/bin/bash", "./imagewrangler.sh", "check" ]

3
images/imagewrangler/post.sh Executable file
View file

@ -0,0 +1,3 @@
#!/usr/bin/env bash
rm -f imagewrangler_ed25519 imagewrangler_ed25519.pub dateetime.txt;

6
images/imagewrangler/pre.sh Executable file
View file

@ -0,0 +1,6 @@
#!/usr/bin/env bash
cp /mnt/shared/imagewrangler/imagewrangler_ed25519 .
cp /mnt/shared/imagewrangler/imagewrangler_ed25519.pub .
date >./datetime.txt

View file

@ -0,0 +1 @@
docker

View file

@ -0,0 +1,10 @@
FROM scratch
ADD ./mapserver /
USER 113:60
WORKDIR /world
# CMD is executed as an argument to ENTRYPOINT.
# Ref https://stackoverflow.com/a/21564990/1460422
ENTRYPOINT ["/mapserver"]

View file

@ -0,0 +1,61 @@
#!/usr/bin/env bash
# The architecture - possible values: arm (default), x86, x86_64
arch="${arch:-arm}";
download_url="https://github.com/minetest-mapserver/mapserver/releases/latest/download/mapserver-linux-${arch}";
# The registry to push to
DOCKER_REGISTRY="${DOCKER_REGISTRY:-registry.service.mooncarrot.space:5000/}";
if [[ "${DOCKER_REGISTRY}" == "hub" ]]; then DOCKER_REGISTRY=""; fi
###############################################################################
# Make sure the current directory is the location of this script to simplify matters
cd "$(dirname "$(readlink -f "$0")")" || { echo "[minetest-mapserver] Error: Failed to cd"; exit 1; };
temp_dir="$(mktemp --tmpdir -d "minetest-mapserver-XXXXXXX")";
on_exit() {
log_msg "Cleaning up";
rm -rf "${temp_dir}";
}
trap on_exit EXIT;
# Ref https://stackoverflow.com/a/911213/1460422
if [ -t 1 ] || [[ ! -z "${FORCE_COLOUR}" ]]; then
###################
# From lantern.sh #
RS="\033[0m" # reset
HC="\033[1m" # hicolor
LC="\033[2m" # locolor / dim
###################
fi
log_msg() {
echo -e "${HC}>>>${RS} ${LC}[ ${SECONDS} ]${RS} $*" >&2;
}
###############################################################################
log_msg "Downloading mapserver";
curl -sSL -o "${temp_dir}/mapserver" "${download_url}"
chmod +x "${temp_dir}/mapserver";
log_msg "Determining version";
version="$(curl -sSL https://api.github.com/repos/minetest-mapserver/mapserver/releases/latest | jq --raw-output .tag_name)";
log_msg "Version is ${version}";
log_msg "Preparing docker build context";
cp ./Dockerfile "${temp_dir}/Dockerfile";
log_msg "Building docker image";
docker_image_name="${DOCKER_REGISTRY}minetest-mapserver:${arch}";
docker build "${temp_dir}" --tag "${docker_image_name}";
log_msg "Pushing to ${DOCKER_REGISTRY}";
docker push "${docker_image_name}";

View file

@ -0,0 +1 @@
base-nopush

View file

@ -0,0 +1,10 @@
ARG REPO_LOCATION
# ARG BASE_VERSION
# NOTE: This might note be what's required - if possible we might only need the Node.js binary.
# Perhaps we could have 2 different images: minideb-node and node-min?
# ANOTHER NOTE: We probably don't need NPM - even in the minideb-node Docker image, as we should be able to npm install outside the container and import it in?
FROM ${REPO_LOCATION}minideb
# FROM ${REPO_LOCATION}minideb:${BASE_VERSION}
RUN install_packages libatomic1 nodejs-sbrl

5
images/minideb-node/delete_npm Executable file
View file

@ -0,0 +1,5 @@
#!/usr/bin/env bash
# Call this script to delete npm.
# This is a self-deleting script, so no need to scrub it afterwards :D
rm -r /usr/local/lib/node/lib/node_modules/npm
rm /usr/local/bin/delete_npm;

View file

@ -0,0 +1 @@
docker

View file

@ -0,0 +1,17 @@
diff --git a/mkimage b/mkimage
index cd9d2c6..ea68301 100755
--- a/mkimage
+++ b/mkimage
@@ -16,10 +16,10 @@ exec 2> >(tee -ia "$LOGFILE" >&2)
DEBOOTSTRAP_DIR=$(mktemp -d)
cp -a /usr/share/debootstrap/* "$DEBOOTSTRAP_DIR"
-cp -a /usr/share/keyrings/debian-archive-keyring.gpg "$DEBOOTSTRAP_DIR"
+cp -a /usr/share/keyrings/raspbian-archive-keyring.gpg "$DEBOOTSTRAP_DIR"
cp -a "${ROOT}/debootstrap/"* "${DEBOOTSTRAP_DIR}/scripts"
-KEYRING=$DEBOOTSTRAP_DIR/debian-archive-keyring.gpg
+KEYRING=$DEBOOTSTRAP_DIR/raspbian-archive-keyring.gpg
if [ -f "${ROOT}/keys/${DIST}.gpg" ]; then
gpg --no-default-keyring --keyring "$KEYRING" --import "${ROOT}/keys/${DIST}.gpg"

142
images/minideb/minideb.sh Executable file
View file

@ -0,0 +1,142 @@
#!/usr/bin/env bash
set -e;
###
# Settings
###
# These can all be changed via environment variables.
clone_url="${clone_url:-https://github.com/bitnami/minideb.git}";
image_version="${image_version:-buster}";
apply_patch_raspbian="${apply_patch_raspbian:-true}";
setup_proxy="${setup_proxy:-true}";
proxy_address="${proxy_address:-http://172.16.230.100:3142}";
###############################################################################
target_directory="${1}";
# Ref https://stackoverflow.com/a/911213/1460422
if [ -t 1 ] || [[ ! -z "${FORCE_COLOUR}" ]]; then
###################
# From lantern.sh #
RS="\033[0m" # reset
HC="\033[1m" # hicolor
LC="\033[2m" # locolor / dim
###################
fi
if [[ -z "${target_directory}" ]]; then
echo "Usage:" >&2;
echo " $0 {target_directory}" >&2;
echo "" >&2;
echo "" >&2;
exit 0;
fi
if [[ "${UID}" -ne "0" ]]; then
echo "Error: This script must be run as root (as we need to chroot to complete the setup)" >&2;
echo "Additional information: You ran this script as ${USER} with uid ${UID}, but we expected uid 0" >&2;
exit 1;
fi
if [[ ! -d "${target_directory}" ]]; then
mkdir -p "${target_directory}";
fi
target_directory="$(realpath "${target_directory}")";
###############################################################################
run_in_chroot() {
if [[ ! -d "${temp_dir_postprocess}" ]] || [[ -z "${temp_dir_postprocess}" ]]; then
echo "Error: Target directory '${temp_dir_postprocess}' does not exist, so can't chroot into it." >&2;
echo "Additional Information: You tried to run '${*}'" >&2;
return 1;
fi
chroot "${temp_dir_postprocess}" "$@";
}
log_msg() {
echo -e "\n${HC}>>>${RS} ${LC}[ ${SECONDS} ]${RS} $*\n"
}
###############################################################################
# Make sure the current directory is the location of this script to simplify matters
cd "$(dirname "$(readlink -f "$0")")";
config_dir="${PWD}";
temp_dir="$(mktemp --tmpdir -d "minideb-build-XXXXXXX")";
temp_dir_postprocess="${temp_dir}/minideb-${image_version}";
temp_dir_build="$(mktemp -d "${config_dir}/minideb-tmp-XXXXXXX")";
on_exit() {
rm -rf "${temp_dir}";
if [[ -e "${temp_dir_build}" ]]; then rm -rf "${temp_dir_build}"; fi
}
trap on_exit EXIT;
###############################################################################
log_msg "Cloning minideb";
git clone "${clone_url}" "${temp_dir}/minideb";
cd "${temp_dir}/minideb";
if [[ "${apply_patch_raspbian}" == "true" ]]; then
log_msg "Applying Raspbian patch";
git apply "${config_dir}/minideb-raspbian.patch";
fi
log_msg "Building minideb base image";
# Must be run as root
time TMPDIR="${temp_dir_build}" make "${image_version}";
cp -r "${temp_dir}/minideb/build" "${temp_dir}/build";
filename_tar="$(find "${temp_dir}/build" -name '*.tar' -print -quit)";
log_msg "Unpacking resulting tar archive";
mkdir "${temp_dir_postprocess}";
tar -xf "${filename_tar}" -C "${temp_dir_postprocess}";
# We're finished with minideb now
rm -rf "${temp_dir}/minideb";
###############################################################################
if [[ "${setup_proxy}" == "true" ]]; then
log_msg "Applying apt proxy settings";
echo "Acquire::http { Proxy \"${proxy_address}\"; }" | run_in_chroot tee /etc/apt/apt.conf.d/proxy
fi
# # Fix /dev/null - ref https://unix.stackexchange.com/a/146639/64687
# run_in_chroot rm -f /dev/null;
# run_in_chroot mknod -m 666 /dev/null c 1 3;
# run_in_chroot mknod -m 666 /dev/zero c 1 5
# run_in_chroot chown root:root /dev/null /dev/zero
# run_in_chroot install_packages gnupg;
log_msg "Adding aptosaurus GPG key";
# Add apt.starbeamrainbowlabs.com
# No need to apt update, because install_packages will do this for us
# Ref https://github.com/bitnami/minideb/blob/6c039b8/mkimage#L181-L206
# Also ref https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=851774
gpg_key_url="https://apt.starbeamrainbowlabs.com/aptosaurus.asc";
echo "deb http://apt.starbeamrainbowlabs.com/ /" | run_in_chroot tee /etc/apt/sources.list.d/sbrl.list;
curl -sSL "${gpg_key_url}" | run_in_chroot sh -c 'cat >/etc/apt/trusted.gpg.d/sbrl-aptosaurus.asc';
###############################################################################
# Repack the tar file
log_msg "Repacking archive";
cd "${temp_dir_postprocess}" || { echo "Failed to cd to unpacked archive directory"; exit 1; };
tar -caf "${target_directory}/minideb.tar.gz" .;
cp "${filename_tar}.log" "${target_directory}/minideb.log";
cp "${filename_tar}.manifest" "${target_directory}/minideb.manifest";
log_msg "Written result to ${target_directory}:";
ls -htFl "${target_directory}";

1
images/minideb/type.txt Normal file
View file

@ -0,0 +1 @@
base