aptosaurus/aptosaurus.sh

#!/usr/bin/env bash

# Make sure the current directory is the location of this script to simplify matters
cd "$(dirname "$(readlink -f "$0")")" || exit 2;
export GNUPGHOME="${PWD}/gnupg";

################
### Settings ###
################

# The name of this project
project_name="aptosaurus";

# The path to the lantern build engine git submodule
lantern_path="./lantern-build-engine";

###
# Custom Settings
###

# The directory that contains the source deb packages
dir_sources="sources";
# The directory the repository is in
dir_repo="repo";

# The id of the GPG key to use for signing
gpg_key_id="7A37B795C20E4651D9BBE1B2D48D801C6A66A5D8";

# Keep the most recent X versions of packages.
# Note that this correctly handles packages that have multiple architectures per version.
keep_versions=3;

###############################################################################

# Check out the lantern git submodule if needed
if [ ! -f "${lantern_path}/lantern.sh" ]; then git submodule update --init "${lantern_path}"; fi
# The lantern build engine is checked elsewhere
# shellcheck source=/dev/null
source "${lantern_path}/lantern.sh";

if [[ "$#" -lt 1 ]]; then
	echo -e "${FBLE}${project_name}${RS}";
	echo -e "    by Starbeamrainbowlabs";
	echo -e "${LC}Powered by the lantern build engine, v${version}${RS}";
	echo -e "";
	echo -e "Based on https://askubuntu.com/a/89698/139735";
	echo -e "";
	echo -e "${CSECTION}Usage${RS}";
	echo -e "    ./build ${CTOKEN}{action}${RS} ${CTOKEN}{action}${RS} ${CTOKEN}{action}${RS} ...";
	echo -e "";
	echo -e "${CSECTION}Available actions${RS}";
	echo -e "    ${CACTION}setup${RS}               - Perform initial setup";
	echo -e "    ${CACTION}update${RS}              - Scan for new packages and add them to the repository";
	echo -e "    ${CACTION}update-cron${RS}         - Like ${CACTION}update${RS}, but silent unless something goes wrong";
	echo -e "    ${CACTION}metafiles${RS}           - Rebuild the repository metafiles only (useful if you've manually fiddled with the repo packages)";
	echo -e "    ${CACTION}generate-summary${RS}    - Generate a SUMMARY.txt file in the repo root";
	echo -e "";
	
	exit 1;
fi

###############################################################################

task_setup() {
	task_begin "Checking environment";
	check_command gpg true;
	check_command cp true;
	check_command rm true;
	check_command mkdir true;
	check_command find true;
	check_command xargs true;
	check_command ln true;
	check_command apt-ftparchive true;
	check_command dpkg-sig true;
	task_end $?;
	
	task_begin "Checking keys";
	if [ ! -f "./public.gpg" ]; then
		echo "Error: Couldn't find the public key to use.";
		echo "Make sure that ${HC}public.gpg${HC} exist on disk.";
		exit 1;
	fi
	task_end $?;
	
	task_begin "Creating directories";
	mkdir "${dir_sources}" "${dir_repo}";
	task_end $?;
	
	if [ ! -d "${GNUPGHOME}" ]; then
		mkdir "${GNUPGHOME}";
		chown 0700 "${GNUPGHOME}";
		if [ ! -f "./secret.gpg" ]; then
			echo "Couldn't find the secret key to use.";
			echo "Make sure ${HC}secret.gpg${RS} exists on disk.";
		fi
		task_begin "Importing keys";
		gpg --import -v -v ./secret.gpg;
		gpg --import -v -v ./public.gpg;
		gpg --list-keys;
		rm secret.gpg;
		task_end $?;
	fi
	
	if [ ! -f "${dir_repo}/aptosaurus.asc" ]; then
		task_begin "Hard linking public signing key";
		cp -al "./public.gpg" "${dir_repo}/aptosaurus.asc";
		task_end $?;
	fi
	
}

# $1 - the .deb file to symlink
_symlink_deb() {
	source="$1";
	destination="$(basename "${source}")";
	
	if [ -f "${destination}" ]; then return; fi
	
	ln -s "${source}" "${destination}";
}

task_update() {
	tasks_run delete-old;
	
	task_begin "Symlinking new packages";
	package_count_before="$(find ${dir_repo} -name "*.deb" | wc -l)";
	export -f _symlink_deb;
	cd "${dir_repo}" || exit 2;
	find "../${dir_sources}" -type f -name "*.deb" -print0 | xargs --null -I{} bash -c '_symlink_deb "{}"';
	exit_code="$?";
	cd - || exit 2;
	package_count_after="$(find ${dir_repo} -name "*.deb" | wc -l)";
	task_end "${exit_code}";
	
	if [[ "${package_count_before}" -eq "${package_count_after}" ]] && [[ -z "${FORCE_UPDATE}" ]]; then
		echo "No new packages to process.";
		exit 0;
	else
		new_package_count=$((package_count_after-package_count_before));
		echo -e "Found ${new_package_count} new packages";
	fi
	
	cd "${dir_repo}" || exit 2;
	
	task_begin "Signing packages";
	execute dpkg-sig -k "${gpg_key_id}" -s builder "*.deb";
	task_end $?;
	
	tasks_run "metafiles";
}

task_metafiles() {
	if [[ "$(basename "${PWD}")" != "$(basename "${dir_repo}")" ]]; then
		cd "${dir_repo}" || { echo "Error: Failed to cd into repo" >&2; exit 3; };
	fi
	task_begin "Building packages file";
	apt-ftparchive packages . >Packages;
	execute bzip2 -kf Packages;
	task_end $?;
	
	task_begin "Generating release file";
	apt-ftparchive release . >Release;
	task_end $?;
	
	task_begin "Signing release file";
	execute gpg --yes -abs -u "${gpg_key_id}" -o Release.gpg Release
	task_end $?;
	
	cd - || { echo "Error: Failed to cd back to previous directory" >&2; exit 4; };
	tasks_run generate-summary;
}

# Spits out a TSV record with 3 columns:
# 1. Package name
# 2. Package version
# 3. Package description
__analyse_package() { dpkg --info "$1" | awk '/^\s*Package:/ { gsub("\\s*Package:\\s*", ""); printf($0 "\t"); } /^\s*Version:/ { gsub("\\s*Version:\\s*", ""); printf($0 "\t"); } /^\s*Description:/ { gsub("\\s*Description:\\s*", ""); print($0); }'; };

# Spits out a TSV record with 2 columns:
# 1. Filename
# 2. Package version
__analyse_package_simple() { dpkg --info "$1" | awk -v filename="$1" 'BEGIN { printf(filename "\t"); } /^\s*Version:/ { gsub("\\s*Version:\\s*", ""); print($0); }'; };


_generate_summary() {
	# xargs: Generate the TSV records in parallel
	# sort: Sort on the package name, then version (note the 2V does a *version sort* on column 2)
	# uniq: Remove duplicates (e.g. due to multiple architectures)
	# column: Align all the columns nicely - ref https://unix.stackexchange.com/a/468048/64687
	
	export -f __analyse_package;
	find "${dir_sources}" -type f -name "*.deb" -print0 | xargs -0 -P "$(nproc)" -I{} bash -c '__analyse_package "{}"' | sort -k1,2V | uniq | column -t -s "$(printf '\t')";
}

task_generate-summary() {
	task_begin "Regenerating SUMMARY.txt";
	_generate_summary >"${dir_repo}/SUMMARY.txt";
	task_end "$?";
}

task_delete-old() {
	task_begin "Deleting packages more than ${keep_versions} versions ago";
	# Find and delete old package versions
	export -f __analyse_package_simple;
	find sources/ -type f -name "*.deb" -print0 | xargs -0 -n1 -P "$(nproc)" -I{} bash -c '__analyse_package_simple "{}"' | sort -k1,2Vr | uniq | awk -v keep_ago=${keep_versions} '{ package=$1; gsub(/^.*\//, "", package); gsub(/_.*+$/, "", package); arch=$1; gsub(/^.*_/, "", arch); gsub(/\.deb$/, "", arch); counts[package arch]++; if(counts[package arch] > keep_ago) print($1); }' | xargs --verbose --no-run-if-empty rm
	# Result resultant broken symlinks
	find "${dir_repo}" -xtype l -delete -print0 | xargs -0 -n1 echo Deleting;
	task_end "$?";
}

task_update-cron() {
	tmpfile="$(mktemp --suffix ".aptosaurus.log")";
	
	set +e; # Allow errors - we're handling them explicitly here
	# Save the output.....
	bash ./aptosaurus.sh update | ansi_strip >"${tmpfile}";
	exit_code="${?}";
	# update *should* do the metafiles too, but it doesn't seem to be doing so when called through cron for some bizarre reason
	bash ./aptosaurus.sh metafiles | ansi_strip >>"${tmpfile}";
	
	# ....but only display it if something went wrong
	if [[ "${exit_code}" -ne 0 ]]; then
		echo "===== Transcript =====";
		cat "${tmpfile}";
		echo "========= end ========";
	fi
	
	rm "${tmpfile}";
}

###############################################################################

tasks_run "$@";