The code that drives my apt repository. https://apt.starbeamrainbowlabs.com
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

234 lines
7.6 KiB

  1. #!/usr/bin/env bash
  2. # Make sure the current directory is the location of this script to simplify matters
  3. cd "$(dirname "$(readlink -f "$0")")" || exit 2;
  4. export GNUPGHOME="${PWD}/gnupg";
  5. ################
  6. ### Settings ###
  7. ################
  8. # The name of this project
  9. project_name="aptosaurus";
  10. # The path to the lantern build engine git submodule
  11. lantern_path="./lantern-build-engine";
  12. ###
  13. # Custom Settings
  14. ###
  15. # The directory that contains the source deb packages
  16. dir_sources="sources";
  17. # The directory the repository is in
  18. dir_repo="repo";
  19. # The id of the GPG key to use for signing
  20. gpg_key_id="7A37B795C20E4651D9BBE1B2D48D801C6A66A5D8";
  21. # Keep the most recent X versions of packages.
  22. # Note that this correctly handles packages that have multiple architectures per version.
  23. keep_versions=3;
  24. ###############################################################################
  25. # Check out the lantern git submodule if needed
  26. if [ ! -f "${lantern_path}/lantern.sh" ]; then git submodule update --init "${lantern_path}"; fi
  27. # The lantern build engine is checked elsewhere
  28. # shellcheck source=/dev/null
  29. source "${lantern_path}/lantern.sh";
  30. if [[ "$#" -lt 1 ]]; then
  31. echo -e "${FBLE}${project_name}${RS}";
  32. echo -e " by Starbeamrainbowlabs";
  33. echo -e "${LC}Powered by the lantern build engine, v${version}${RS}";
  34. echo -e "";
  35. echo -e "Based on https://askubuntu.com/a/89698/139735";
  36. echo -e "";
  37. echo -e "${CSECTION}Usage${RS}";
  38. echo -e " ./build ${CTOKEN}{action}${RS} ${CTOKEN}{action}${RS} ${CTOKEN}{action}${RS} ...";
  39. echo -e "";
  40. echo -e "${CSECTION}Available actions${RS}";
  41. echo -e " ${CACTION}setup${RS} - Perform initial setup";
  42. echo -e " ${CACTION}update${RS} - Scan for new packages and add them to the repository";
  43. echo -e " ${CACTION}update-cron${RS} - Like ${CACTION}update${RS}, but silent unless something goes wrong";
  44. echo -e " ${CACTION}metafiles${RS} - Rebuild the repository metafiles only (useful if you've manually fiddled with the repo packages)";
  45. echo -e " ${CACTION}generate-summary${RS} - Generate a SUMMARY.txt file in the repo root";
  46. echo -e "";
  47. exit 1;
  48. fi
  49. ###############################################################################
  50. task_setup() {
  51. task_begin "Checking environment";
  52. check_command gpg true;
  53. check_command cp true;
  54. check_command rm true;
  55. check_command mkdir true;
  56. check_command find true;
  57. check_command xargs true;
  58. check_command ln true;
  59. check_command apt-ftparchive true;
  60. check_command dpkg-sig true;
  61. task_end $?;
  62. task_begin "Checking keys";
  63. if [ ! -f "./public.gpg" ]; then
  64. echo "Error: Couldn't find the public key to use.";
  65. echo "Make sure that ${HC}public.gpg${HC} exist on disk.";
  66. exit 1;
  67. fi
  68. task_end $?;
  69. task_begin "Creating directories";
  70. mkdir "${dir_sources}" "${dir_repo}";
  71. task_end $?;
  72. if [ ! -d "${GNUPGHOME}" ]; then
  73. mkdir "${GNUPGHOME}";
  74. chown 0700 "${GNUPGHOME}";
  75. if [ ! -f "./secret.gpg" ]; then
  76. echo "Couldn't find the secret key to use.";
  77. echo "Make sure ${HC}secret.gpg${RS} exists on disk.";
  78. fi
  79. task_begin "Importing keys";
  80. gpg --import -v -v ./secret.gpg;
  81. gpg --import -v -v ./public.gpg;
  82. gpg --list-keys;
  83. rm secret.gpg;
  84. task_end $?;
  85. fi
  86. if [ ! -f "${dir_repo}/aptosaurus.asc" ]; then
  87. task_begin "Hard linking public signing key";
  88. cp -al "./public.gpg" "${dir_repo}/aptosaurus.asc";
  89. task_end $?;
  90. fi
  91. }
  92. # $1 - the .deb file to symlink
  93. _symlink_deb() {
  94. source="$1";
  95. destination="$(basename "${source}")";
  96. if [ -f "${destination}" ]; then return; fi
  97. ln -s "${source}" "${destination}";
  98. }
  99. task_update() {
  100. tasks_run delete-old;
  101. task_begin "Symlinking new packages";
  102. package_count_before="$(find ${dir_repo} -name "*.deb" | wc -l)";
  103. export -f _symlink_deb;
  104. cd "${dir_repo}" || exit 2;
  105. find "../${dir_sources}" -type f -name "*.deb" -print0 | xargs --null -n1 -I{} bash -c '_symlink_deb "{}"';
  106. exit_code="$?";
  107. cd - || exit 2;
  108. package_count_after="$(find ${dir_repo} -name "*.deb" | wc -l)";
  109. task_end "${exit_code}";
  110. if [[ "${package_count_before}" -eq "${package_count_after}" ]] && [[ -z "${FORCE_UPDATE}" ]]; then
  111. echo "No new packages to process.";
  112. exit 0;
  113. else
  114. new_package_count=$((package_count_after-package_count_before));
  115. echo -e "Found ${new_package_count} new packages";
  116. fi
  117. cd "${dir_repo}" || exit 2;
  118. task_begin "Signing packages";
  119. execute dpkg-sig -k "${gpg_key_id}" -s builder "*.deb";
  120. task_end $?;
  121. tasks_run "metafiles";
  122. }
  123. task_metafiles() {
  124. if [[ "$(basename "${PWD}")" != "$(basename "${dir_repo}")" ]]; then
  125. cd "${dir_repo}" || { echo "Error: Failed to cd into repo" >&2; exit 3; };
  126. fi
  127. task_begin "Building packages file";
  128. apt-ftparchive packages . >Packages;
  129. execute bzip2 -kf Packages;
  130. task_end $?;
  131. task_begin "Generating release file";
  132. apt-ftparchive release . >Release;
  133. task_end $?;
  134. task_begin "Signing release file";
  135. execute gpg --yes -abs -u "${gpg_key_id}" -o Release.gpg Release
  136. task_end $?;
  137. cd - || { echo "Error: Failed to cd back to previous directory" >&2; exit 4; };
  138. tasks_run generate-summary;
  139. }
  140. # Spits out a TSV record with 3 columns:
  141. # 1. Package name
  142. # 2. Package version
  143. # 3. Package description
  144. __analyse_package() { dpkg --info "$1" | awk '/^\s*Package:/ { gsub("\\s*Package:\\s*", ""); printf($0 "\t"); } /^\s*Version:/ { gsub("\\s*Version:\\s*", ""); printf($0 "\t"); } /^\s*Description:/ { gsub("\\s*Description:\\s*", ""); print($0); }'; };
  145. # Spits out a TSV record with 2 columns:
  146. # 1. Filename
  147. # 2. Package version
  148. __analyse_package_simple() { dpkg --info "$1" | awk -v filename="$1" 'BEGIN { printf(filename "\t"); } /^\s*Version:/ { gsub("\\s*Version:\\s*", ""); print($0); }'; };
  149. _generate_summary() {
  150. # xargs: Generate the TSV records in parallel
  151. # sort: Sort on the package name, then version (note the 2V does a *version sort* on column 2)
  152. # uniq: Remove duplicates (e.g. due to multiple architectures)
  153. # column: Align all the columns nicely - ref https://unix.stackexchange.com/a/468048/64687
  154. export -f __analyse_package;
  155. find "${dir_sources}" -type f -name "*.deb" -print0 | xargs -0 -n1 -P "$(nproc)" -I{} bash -c '__analyse_package "{}"' | sort -k1,2V | uniq | column -t -s "$(printf '\t')";
  156. }
  157. task_generate-summary() {
  158. task_begin "Regenerating SUMMARY.txt";
  159. _generate_summary >"${dir_repo}/SUMMARY.txt";
  160. task_end "$?";
  161. }
  162. task_delete-old() {
  163. task_begin "Deleting packages more than ${keep_versions} versions ago";
  164. # Find and delete old package versions
  165. export -f __analyse_package_simple;
  166. find sources/ -type f -name "*.deb" -print0 | xargs -0 -n1 -P "$(nproc)" -I{} bash -c '__analyse_package_simple "{}"' | sort -k1,2Vr | uniq | awk -v keep_ago=${keep_versions} '{ package=$1; gsub(/^.*\//, "", package); gsub(/_.*+$/, "", package); arch=$1; gsub(/^.*_/, "", arch); gsub(/\.deb$/, "", arch); counts[package arch]++; if(counts[package arch] > keep_ago) print($1); }' | xargs --verbose --no-run-if-empty rm
  167. # Result resultant broken symlinks
  168. find "${dir_repo}" -xtype l -delete -print0 | xargs -0 -n1 echo Deleting;
  169. task_end "$?";
  170. }
  171. task_update-cron() {
  172. tmpfile="$(mktemp --suffix ".aptosaurus.log")";
  173. set +e; # Allow errors - we're handling them explicitly here
  174. # Save the output.....
  175. bash ./aptosaurus.sh update | ansi_strip >"${tmpfile}";
  176. exit_code="${?}";
  177. # update *should* do the metafiles too, but it doesn't seem to be doing so when called through cron for some bizarre reason
  178. bash ./aptosaurus.sh metafiles | ansi_strip >>"${tmpfile}";
  179. # ....but only display it if something went wrong
  180. if [[ "${exit_code}" -ne 0 ]]; then
  181. echo "===== Transcript =====";
  182. cat "${tmpfile}";
  183. echo "========= end ========";
  184. fi
  185. rm "${tmpfile}";
  186. }
  187. ###############################################################################
  188. tasks_run "$@";