|
|
|
@ -1,5 +1,38 @@ |
|
|
|
|
#!/usr/bin/env bash |
|
|
|
|
|
|
|
|
|
### MediaWiki helpers ### |
|
|
|
|
|
|
|
|
|
# Iterates over all the pages in a category. |
|
|
|
|
# Handles continues (multi-request page lists) correctly. |
|
|
|
|
# $1 - Wiki api.php root |
|
|
|
|
# $2 - Category name |
|
|
|
|
function iterate_category() { |
|
|
|
|
# Example url: https://starbounder.org/mediawiki/api.php?action=query&format=json&list=categorymembers&cmlimit=max&cmtitle=Category:Craftables |
|
|
|
|
initial_url="$1?action=query&format=json&list=categorymembers&cmlimit=max&cmtitle=$2"; |
|
|
|
|
next_url="${initial_url}"; |
|
|
|
|
|
|
|
|
|
temp_file="$(mktemp --suffix .CategoryDownloaded.json)"; |
|
|
|
|
|
|
|
|
|
echo "Iterating over category $2 at $1" >&2; |
|
|
|
|
echo "Temporary file is at ${temp_file}" >&2; |
|
|
|
|
|
|
|
|
|
while :; do |
|
|
|
|
echo "Fetching ${next_url}" >&2; |
|
|
|
|
curl -sS -o "${temp_file}" "${next_url}"; |
|
|
|
|
|
|
|
|
|
jq --raw-output '.query.categorymembers[].title' <"${temp_file}" |
|
|
|
|
|
|
|
|
|
# If there's no continue object in the response, then we must be done |
|
|
|
|
if [ "$(cat "${temp_file}" | jq --raw-output '.continue' --monochrome-output)" == "null" ]; then |
|
|
|
|
echo "Category iteration complete" >&2; |
|
|
|
|
break |
|
|
|
|
fi |
|
|
|
|
|
|
|
|
|
continue_code="$(cat "${temp_file}" | jq --raw-output '.continue.cmcontinue')"; |
|
|
|
|
next_url="${initial_url}&cmcontinue=${continue_code}"; |
|
|
|
|
done |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
### CrossCode ### |
|
|
|
|
curl https://crosscode.gamepedia.com/Items | xidel --data - --css "table a" | awk "NF > 0" | sort >Cross-Code-Items.txt |
|
|
|
|
|
|
|
|
@ -37,6 +70,12 @@ function list_categories() { |
|
|
|
|
|
|
|
|
|
list_pages "Category:Dragons" >Dragons.txt |
|
|
|
|
|
|
|
|
|
### Starbound ### |
|
|
|
|
curl https://starbounder.org/Blocks | xidel --data - --css ".gametable a" | awk '/[^.].*/' | sort | uniq >Starbound.txt |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
iterate_category "https://starbounder.org/mediawiki/api.php" "Category:Craftables" >>Starbound.txt; |
|
|
|
|
|
|
|
|
|
### Shakespeare's Complete Works ### |
|
|
|
|
curl https://www.gutenberg.org/files/100/100-0.txt >Shakespeares-Works.txt |
|
|
|
|