aptosaurus/aptosaurus.sh

234 lines
7.6 KiB
Bash
Raw Normal View History

2019-08-05 23:02:44 +00:00
#!/usr/bin/env bash
# Make sure the current directory is the location of this script to simplify matters
cd "$(dirname "$(readlink -f "$0")")" || exit 2;
2019-08-05 23:33:58 +00:00
export GNUPGHOME="${PWD}/gnupg";
2019-08-05 23:02:44 +00:00
################
### Settings ###
################
# The name of this project
project_name="aptosaurus";
# The path to the lantern build engine git submodule
lantern_path="./lantern-build-engine";
###
# Custom Settings
###
# The directory that contains the source deb packages
dir_sources="sources";
# The directory the repository is in
dir_repo="repo";
# The id of the GPG key to use for signing
gpg_key_id="7A37B795C20E4651D9BBE1B2D48D801C6A66A5D8";
# Keep the most recent X versions of packages.
# Note that this correctly handles packages that have multiple architectures per version.
keep_versions=3;
2019-08-05 23:02:44 +00:00
###############################################################################
# Check out the lantern git submodule if needed
if [ ! -f "${lantern_path}/lantern.sh" ]; then git submodule update --init "${lantern_path}"; fi
2020-03-23 01:53:14 +00:00
# The lantern build engine is checked elsewhere
# shellcheck source=/dev/null
2019-08-05 23:02:44 +00:00
source "${lantern_path}/lantern.sh";
if [[ "$#" -lt 1 ]]; then
echo -e "${FBLE}${project_name}${RS}";
echo -e " by Starbeamrainbowlabs";
echo -e "${LC}Powered by the lantern build engine, v${version}${RS}";
echo -e "";
echo -e "Based on https://askubuntu.com/a/89698/139735";
echo -e "";
echo -e "${CSECTION}Usage${RS}";
echo -e " ./build ${CTOKEN}{action}${RS} ${CTOKEN}{action}${RS} ${CTOKEN}{action}${RS} ...";
echo -e "";
echo -e "${CSECTION}Available actions${RS}";
2020-03-23 01:38:56 +00:00
echo -e " ${CACTION}setup${RS} - Perform initial setup";
echo -e " ${CACTION}update${RS} - Scan for new packages and add them to the repository";
echo -e " ${CACTION}update-cron${RS} - Like ${CACTION}update${RS}, but silent unless something goes wrong";
echo -e " ${CACTION}metafiles${RS} - Rebuild the repository metafiles only (useful if you've manually fiddled with the repo packages)";
echo -e " ${CACTION}generate-summary${RS} - Generate a SUMMARY.txt file in the repo root";
2019-08-05 23:02:44 +00:00
echo -e "";
exit 1;
fi
###############################################################################
task_setup() {
task_begin "Checking environment";
check_command gpg true;
check_command cp true;
check_command rm true;
check_command mkdir true;
check_command find true;
check_command xargs true;
check_command ln true;
check_command apt-ftparchive true;
check_command dpkg-sig true;
task_end $?;
task_begin "Checking keys";
if [ ! -f "./public.gpg" ]; then
echo "Error: Couldn't find the public key to use.";
echo "Make sure that ${HC}public.gpg${HC} exist on disk.";
exit 1;
fi
task_end $?;
task_begin "Creating directories";
2019-08-05 23:41:43 +00:00
mkdir "${dir_sources}" "${dir_repo}";
task_end $?;
2019-08-05 23:41:43 +00:00
if [ ! -d "${GNUPGHOME}" ]; then
2019-08-05 23:44:05 +00:00
mkdir "${GNUPGHOME}";
chown 0700 "${GNUPGHOME}";
2019-08-05 23:02:44 +00:00
if [ ! -f "./secret.gpg" ]; then
echo "Couldn't find the secret key to use.";
echo "Make sure ${HC}secret.gpg${RS} exists on disk.";
fi
task_begin "Importing keys";
gpg --import -v -v ./secret.gpg;
gpg --import -v -v ./public.gpg;
gpg --list-keys;
rm secret.gpg;
task_end $?;
fi
2020-03-23 01:53:14 +00:00
if [ ! -f "${dir_repo}/aptosaurus.asc" ]; then
2019-08-05 23:02:44 +00:00
task_begin "Hard linking public signing key";
cp -al "./public.gpg" "${dir_repo}/aptosaurus.asc";
task_end $?;
fi
}
# $1 - the .deb file to symlink
_symlink_deb() {
source="$1";
2019-08-05 23:49:46 +00:00
destination="$(basename "${source}")";
2019-08-05 23:02:44 +00:00
2019-08-05 23:49:46 +00:00
if [ -f "${destination}" ]; then return; fi
2019-08-05 23:02:44 +00:00
2019-08-05 23:49:46 +00:00
ln -s "${source}" "${destination}";
2019-08-05 23:02:44 +00:00
}
task_update() {
tasks_run delete-old;
2019-08-05 23:02:44 +00:00
task_begin "Symlinking new packages";
2019-08-05 23:51:50 +00:00
package_count_before="$(find ${dir_repo} -name "*.deb" | wc -l)";
2019-08-05 23:02:44 +00:00
export -f _symlink_deb;
cd "${dir_repo}" || exit 2;
2023-06-26 21:07:18 +00:00
find "../${dir_sources}" -type f -name "*.deb" -print0 | xargs --null -I{} bash -c '_symlink_deb "{}"';
2019-08-05 23:02:44 +00:00
exit_code="$?";
cd - || exit 2;
2019-08-05 23:51:50 +00:00
package_count_after="$(find ${dir_repo} -name "*.deb" | wc -l)";
2019-08-05 23:02:44 +00:00
task_end "${exit_code}";
2020-05-13 19:27:04 +00:00
if [[ "${package_count_before}" -eq "${package_count_after}" ]] && [[ -z "${FORCE_UPDATE}" ]]; then
2019-08-05 23:02:44 +00:00
echo "No new packages to process.";
exit 0;
else
new_package_count=$((package_count_after-package_count_before));
echo -e "Found ${new_package_count} new packages";
fi
cd "${dir_repo}" || exit 2;
task_begin "Signing packages";
execute dpkg-sig -k "${gpg_key_id}" -s builder "*.deb";
task_end $?;
tasks_run "metafiles";
}
task_metafiles() {
if [[ "$(basename "${PWD}")" != "$(basename "${dir_repo}")" ]]; then
cd "${dir_repo}" || { echo "Error: Failed to cd into repo" >&2; exit 3; };
fi
2019-08-05 23:02:44 +00:00
task_begin "Building packages file";
2019-08-06 00:21:29 +00:00
apt-ftparchive packages . >Packages;
2019-08-05 23:02:44 +00:00
execute bzip2 -kf Packages;
task_end $?;
2019-08-05 23:52:49 +00:00
task_begin "Generating release file";
2019-08-06 00:21:29 +00:00
apt-ftparchive release . >Release;
2019-08-05 23:52:49 +00:00
task_end $?;
2019-08-05 23:02:44 +00:00
2019-08-05 23:52:49 +00:00
task_begin "Signing release file";
2019-08-05 23:02:44 +00:00
execute gpg --yes -abs -u "${gpg_key_id}" -o Release.gpg Release
task_end $?;
2020-03-23 01:41:59 +00:00
cd - || { echo "Error: Failed to cd back to previous directory" >&2; exit 4; };
2020-03-23 01:41:59 +00:00
tasks_run generate-summary;
2019-08-05 23:02:44 +00:00
}
2020-03-23 01:38:56 +00:00
# Spits out a TSV record with 3 columns:
# 1. Package name
# 2. Package version
# 3. Package description
__analyse_package() { dpkg --info "$1" | awk '/^\s*Package:/ { gsub("\\s*Package:\\s*", ""); printf($0 "\t"); } /^\s*Version:/ { gsub("\\s*Version:\\s*", ""); printf($0 "\t"); } /^\s*Description:/ { gsub("\\s*Description:\\s*", ""); print($0); }'; };
# Spits out a TSV record with 2 columns:
# 1. Filename
# 2. Package version
__analyse_package_simple() { dpkg --info "$1" | awk -v filename="$1" 'BEGIN { printf(filename "\t"); } /^\s*Version:/ { gsub("\\s*Version:\\s*", ""); print($0); }'; };
2020-03-23 01:38:56 +00:00
_generate_summary() {
2020-03-23 01:44:55 +00:00
# xargs: Generate the TSV records in parallel
# sort: Sort on the package name, then version (note the 2V does a *version sort* on column 2)
# uniq: Remove duplicates (e.g. due to multiple architectures)
# column: Align all the columns nicely - ref https://unix.stackexchange.com/a/468048/64687
2020-03-23 01:38:56 +00:00
export -f __analyse_package;
2023-06-26 21:07:18 +00:00
find "${dir_sources}" -type f -name "*.deb" -print0 | xargs -0 -P "$(nproc)" -I{} bash -c '__analyse_package "{}"' | sort -k1,2V | uniq | column -t -s "$(printf '\t')";
2020-03-23 01:38:56 +00:00
}
task_generate-summary() {
2020-03-23 01:41:59 +00:00
task_begin "Regenerating SUMMARY.txt";
2020-03-23 01:38:56 +00:00
_generate_summary >"${dir_repo}/SUMMARY.txt";
2020-03-23 01:41:59 +00:00
task_end "$?";
2020-03-23 01:38:56 +00:00
}
task_delete-old() {
task_begin "Deleting packages more than ${keep_versions} versions ago";
# Find and delete old package versions
2020-05-13 19:32:03 +00:00
export -f __analyse_package_simple;
2024-02-01 02:05:25 +00:00
find sources/ -type f -name "*.deb" -print0 | xargs -0 -P "$(nproc)" -I{} bash -c '__analyse_package_simple "{}"' | sort -k1,2Vr | uniq | awk -v keep_ago=${keep_versions} '{ package=$1; gsub(/^.*\//, "", package); gsub(/_.*+$/, "", package); arch=$1; gsub(/^.*_/, "", arch); gsub(/\.deb$/, "", arch); counts[package arch]++; if(counts[package arch] > keep_ago) print($1); }' | xargs --verbose --no-run-if-empty rm
# Result resultant broken symlinks
find "${dir_repo}" -xtype l -delete -print0 | xargs -0 -n1 echo Deleting;
task_end "$?";
}
2019-08-09 22:45:32 +00:00
task_update-cron() {
2019-08-09 22:46:57 +00:00
tmpfile="$(mktemp --suffix ".aptosaurus.log")";
2019-08-09 22:45:32 +00:00
2020-03-23 01:41:59 +00:00
set +e; # Allow errors - we're handling them explicitly here
# Save the output.....
2019-08-09 22:48:00 +00:00
bash ./aptosaurus.sh update | ansi_strip >"${tmpfile}";
2019-08-09 22:50:17 +00:00
exit_code="${?}";
2020-07-03 17:15:00 +00:00
# update *should* do the metafiles too, but it doesn't seem to be doing so when called through cron for some bizarre reason
bash ./aptosaurus.sh metafiles | ansi_strip >>"${tmpfile}";
2019-08-09 22:45:32 +00:00
2020-03-23 01:41:59 +00:00
# ....but only display it if something went wrong
2019-08-09 22:45:32 +00:00
if [[ "${exit_code}" -ne 0 ]]; then
echo "===== Transcript =====";
2019-08-09 22:45:32 +00:00
cat "${tmpfile}";
echo "========= end ========";
2019-08-09 22:45:32 +00:00
fi
rm "${tmpfile}";
2019-08-09 22:45:32 +00:00
}
2019-08-05 23:02:44 +00:00
###############################################################################
2020-03-23 01:53:14 +00:00
tasks_run "$@";