Compare commits
No commits in common. "55e3a96fe0cecdd91292c5fcb4a2cd418c40a2a4" and "b5210030bd387a8c1722b51d6b7f18bcf98235fd" have entirely different histories.
55e3a96fe0
...
b5210030bd
3 changed files with 0 additions and 1415 deletions
|
@ -38,7 +38,6 @@ Filename | Contents | Comments
|
||||||
`Cross-Code-Items.txt` | List of items in _[Cross-Code](http://cross-code.com/)_ | A list of all items in the game _CrossCode_. Scraped from [here](https://crosscode.gamepedia.com/Items) by a clever bit of bash in `wordlists/download.sh`. I obviously don't own any of these names.
|
`Cross-Code-Items.txt` | List of items in _[Cross-Code](http://cross-code.com/)_ | A list of all items in the game _CrossCode_. Scraped from [here](https://crosscode.gamepedia.com/Items) by a clever bit of bash in `wordlists/download.sh`. I obviously don't own any of these names.
|
||||||
`Final-Fantasy-15-Items.txt` | List of items in _[Final Fantasy 15](https://finalfantasyxv.square-enix-games.com/)_ | Another list of in-game items - this time from _Final Fantasy XV_. Scraped from [this wiki page](http://finalfantasy.wikia.com/wiki/List_of_Final_Fantasy_XV_items) and related pages. The even cleverer bit of bash that does this is also in `wordlists/download.sh`. Again, I don't own any of these :-)
|
`Final-Fantasy-15-Items.txt` | List of items in _[Final Fantasy 15](https://finalfantasyxv.square-enix-games.com/)_ | Another list of in-game items - this time from _Final Fantasy XV_. Scraped from [this wiki page](http://finalfantasy.wikia.com/wiki/List_of_Final_Fantasy_XV_items) and related pages. The even cleverer bit of bash that does this is also in `wordlists/download.sh`. Again, I don't own any of these :-)
|
||||||
`No-Mans-Sky-Items.txt` | List of items in [No Man's Sky](https://www.nomanssky.com/) | Yep, you guessed it. Don't own this. Bash available in `wordlists/download.sh` - I had a bit of trouble with this one, and had to use an awkward hack or two.
|
`No-Mans-Sky-Items.txt` | List of items in [No Man's Sky](https://www.nomanssky.com/) | Yep, you guessed it. Don't own this. Bash available in `wordlists/download.sh` - I had a bit of trouble with this one, and had to use an awkward hack or two.
|
||||||
`Starbound.txt` | List of blocks and items in [Starbound](https://playstarbound.com/) | From the [Official Wiki](https://starbounder.org/) - I don't own it, I just wrote the scraper :P
|
|
||||||
|
|
||||||
## Credits
|
## Credits
|
||||||
- The code was written by [Starbeamrainbowlabs](https://starbeamrainbowlabs.com/) - that's me too!
|
- The code was written by [Starbeamrainbowlabs](https://starbeamrainbowlabs.com/) - that's me too!
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -1,38 +1,5 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
### MediaWiki helpers ###
|
|
||||||
|
|
||||||
# Iterates over all the pages in a category.
|
|
||||||
# Handles continues (multi-request page lists) correctly.
|
|
||||||
# $1 - Wiki api.php root
|
|
||||||
# $2 - Category name
|
|
||||||
function iterate_category() {
|
|
||||||
# Example url: https://starbounder.org/mediawiki/api.php?action=query&format=json&list=categorymembers&cmlimit=max&cmtitle=Category:Craftables
|
|
||||||
initial_url="$1?action=query&format=json&list=categorymembers&cmlimit=max&cmtitle=$2";
|
|
||||||
next_url="${initial_url}";
|
|
||||||
|
|
||||||
temp_file="$(mktemp --suffix .CategoryDownloaded.json)";
|
|
||||||
|
|
||||||
echo "Iterating over category $2 at $1" >&2;
|
|
||||||
echo "Temporary file is at ${temp_file}" >&2;
|
|
||||||
|
|
||||||
while :; do
|
|
||||||
echo "Fetching ${next_url}" >&2;
|
|
||||||
curl -sS -o "${temp_file}" "${next_url}";
|
|
||||||
|
|
||||||
jq --raw-output '.query.categorymembers[].title' <"${temp_file}"
|
|
||||||
|
|
||||||
# If there's no continue object in the response, then we must be done
|
|
||||||
if [ "$(cat "${temp_file}" | jq --raw-output '.continue' --monochrome-output)" == "null" ]; then
|
|
||||||
echo "Category iteration complete" >&2;
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
|
|
||||||
continue_code="$(cat "${temp_file}" | jq --raw-output '.continue.cmcontinue')";
|
|
||||||
next_url="${initial_url}&cmcontinue=${continue_code}";
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
### CrossCode ###
|
### CrossCode ###
|
||||||
curl https://crosscode.gamepedia.com/Items | xidel --data - --css "table a" | awk "NF > 0" | sort >Cross-Code-Items.txt
|
curl https://crosscode.gamepedia.com/Items | xidel --data - --css "table a" | awk "NF > 0" | sort >Cross-Code-Items.txt
|
||||||
|
|
||||||
|
@ -70,12 +37,6 @@ function list_categories() {
|
||||||
|
|
||||||
list_pages "Category:Dragons" >Dragons.txt
|
list_pages "Category:Dragons" >Dragons.txt
|
||||||
|
|
||||||
### Starbound ###
|
|
||||||
curl https://starbounder.org/Blocks | xidel --data - --css ".gametable a" | awk '/[^.].*/' | sort | uniq >Starbound.txt
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
iterate_category "https://starbounder.org/mediawiki/api.php" "Category:Craftables" >>Starbound.txt;
|
|
||||||
|
|
||||||
### Shakespeare's Complete Works ###
|
### Shakespeare's Complete Works ###
|
||||||
curl https://www.gutenberg.org/files/100/100-0.txt >Shakespeares-Works.txt
|
curl https://www.gutenberg.org/files/100/100-0.txt >Shakespeares-Works.txt
|
||||||
|
|
Loading…
Reference in a new issue