Add Starbound blocks and items list

wordlists/download.sh

@@ -1,5 +1,38 @@
 #!/usr/bin/env bash
 
+### MediaWiki helpers ###
+
+# Iterates over all the pages in a category.
+# Handles continues (multi-request page lists) correctly.
+# $1 - Wiki api.php root
+# $2 - Category name
+function iterate_category() {
+	# Example url: https://starbounder.org/mediawiki/api.php?action=query&format=json&list=categorymembers&cmlimit=max&cmtitle=Category:Craftables
+	initial_url="$1?action=query&format=json&list=categorymembers&cmlimit=max&cmtitle=$2";
+	next_url="${initial_url}";
+	
+	temp_file="$(mktemp --suffix .CategoryDownloaded.json)";
+	
+	echo "Iterating over category $2 at $1" >&2;
+	echo "Temporary file is at ${temp_file}" >&2;
+	
+	while :; do
+		echo "Fetching ${next_url}" >&2;
+		curl -sS -o "${temp_file}" "${next_url}";
+		
+		jq --raw-output '.query.categorymembers[].title' <"${temp_file}"
+		
+		# If there's no continue object in the response, then we must be done
+		if [ "$(cat "${temp_file}" | jq --raw-output '.continue' --monochrome-output)" == "null" ]; then
+			echo "Category iteration complete" >&2;
+			break
+		fi
+		
+		continue_code="$(cat "${temp_file}" | jq --raw-output '.continue.cmcontinue')";
+		next_url="${initial_url}&cmcontinue=${continue_code}";
+	done
+}
+
 ### CrossCode ###
 curl https://crosscode.gamepedia.com/Items | xidel --data - --css "table a" | awk "NF > 0" | sort >Cross-Code-Items.txt
 
@@ -37,6 +70,12 @@ function list_categories() {
 
 list_pages "Category:Dragons" >Dragons.txt
 
+### Starbound ###
+curl https://starbounder.org/Blocks | xidel --data - --css ".gametable a" | awk '/[^.].*/' | sort | uniq >Starbound.txt
+
+
+
+iterate_category "https://starbounder.org/mediawiki/api.php" "Category:Craftables" >>Starbound.txt;
 
 ### Shakespeare's Complete Works ###
 curl https://www.gutenberg.org/files/100/100-0.txt >Shakespeares-Works.txt