Setup scaffolding for Node.js based attempt

This commit is contained in:
Starbeamrainbowlabs 2021-10-01 02:44:50 +01:00
parent cf3d3355ca
commit ab1e52417d
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
10 changed files with 572 additions and 0 deletions

160
.gitignore vendored
View file

@ -35,3 +35,163 @@ Cargo.lock
*.pdb *.pdb
# End of https://www.toptal.com/developers/gitignore/api/rust,git # End of https://www.toptal.com/developers/gitignore/api/rust,git
# Created by https://www.toptal.com/developers/gitignore/api/rust,node,git
# Edit at https://www.toptal.com/developers/gitignore?templates=rust,node,git
### Git ###
# Created by git for backups. To disable backups in Git:
# $ git config --global mergetool.keepBackup false
*.orig
# Created by git when using merge tools for conflicts
*.BACKUP.*
*.BASE.*
*.LOCAL.*
*.REMOTE.*
*_BACKUP_*.txt
*_BASE_*.txt
*_LOCAL_*.txt
*_REMOTE_*.txt
### Node ###
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
.env.production
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
### Node Patch ###
# Serverless Webpack directories
.webpack/
### Rust ###
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
Cargo.lock
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
# End of https://www.toptal.com/developers/gitignore/api/rust,node,git

275
package-lock.json generated Normal file
View file

@ -0,0 +1,275 @@
{
"name": "systemquery",
"version": "1.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "systemquery",
"version": "1.0.0",
"license": "MPL-2.0",
"dependencies": {
"applause-cli": "^1.7.0",
"log": "^6.2.0",
"make-cert": "^1.2.1",
"tweetnacl": "^1.0.3"
}
},
"node_modules/applause-cli": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/applause-cli/-/applause-cli-1.7.0.tgz",
"integrity": "sha512-zO/nBR9x37Iqlm3R2Glo0NyDIevwbDPrISprK61Z/M01Xvm5dFkL2FZAONxxHctT9QoF71fiY22sFRuq8Sqxug=="
},
"node_modules/d": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
"integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==",
"dependencies": {
"es5-ext": "^0.10.50",
"type": "^1.0.1"
}
},
"node_modules/d/node_modules/type": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz",
"integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg=="
},
"node_modules/duration": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/duration/-/duration-0.2.2.tgz",
"integrity": "sha512-06kgtea+bGreF5eKYgI/36A6pLXggY7oR4p1pq4SmdFBn1ReOL5D8RhG64VrqfTTKNucqqtBAwEj8aB88mcqrg==",
"dependencies": {
"d": "1",
"es5-ext": "~0.10.46"
}
},
"node_modules/es5-ext": {
"version": "0.10.53",
"resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz",
"integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==",
"dependencies": {
"es6-iterator": "~2.0.3",
"es6-symbol": "~3.1.3",
"next-tick": "~1.0.0"
}
},
"node_modules/es6-iterator": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
"integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=",
"dependencies": {
"d": "1",
"es5-ext": "^0.10.35",
"es6-symbol": "^3.1.1"
}
},
"node_modules/es6-symbol": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz",
"integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==",
"dependencies": {
"d": "^1.0.1",
"ext": "^1.1.2"
}
},
"node_modules/event-emitter": {
"version": "0.3.5",
"resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
"integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=",
"dependencies": {
"d": "1",
"es5-ext": "~0.10.14"
}
},
"node_modules/ext": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/ext/-/ext-1.6.0.tgz",
"integrity": "sha512-sdBImtzkq2HpkdRLtlLWDa6w4DX22ijZLKx8BMPUuKe1c5lbN6xwQDQCxSfxBQnHZ13ls/FH0MQZx/q/gr6FQg==",
"dependencies": {
"type": "^2.5.0"
}
},
"node_modules/log": {
"version": "6.2.0",
"resolved": "https://registry.npmjs.org/log/-/log-6.2.0.tgz",
"integrity": "sha512-W1sDY5FqR6wlpygW8ZFSxCfBhKx/RzCHK5S+Br8zA14bAnwSgCm5hToIWzi0Yhy6x9Ppw7pyIV06r8F5cSRHUw==",
"dependencies": {
"d": "^1.0.1",
"duration": "^0.2.2",
"es5-ext": "^0.10.53",
"event-emitter": "^0.3.5",
"sprintf-kit": "^2.0.1",
"type": "^2.5.0"
}
},
"node_modules/make-cert": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/make-cert/-/make-cert-1.2.1.tgz",
"integrity": "sha512-mvQkS2rSZE0rfgxycg/g2gcFaNm/XzDgkDF3CIp9StDaPU8f+3NL+gwg/qzsgL8bDy50jwak6OonVrAc/U/xEw==",
"dependencies": {
"node-forge": "^0.10.0"
},
"bin": {
"make-cert": "make-cert.js"
}
},
"node_modules/next-tick": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz",
"integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw="
},
"node_modules/node-forge": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",
"integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==",
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/sprintf-kit": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/sprintf-kit/-/sprintf-kit-2.0.1.tgz",
"integrity": "sha512-2PNlcs3j5JflQKcg4wpdqpZ+AjhQJ2OZEo34NXDtlB0tIPG84xaaXhpA8XFacFiwjKA4m49UOYG83y3hbMn/gQ==",
"dependencies": {
"es5-ext": "^0.10.53"
}
},
"node_modules/tweetnacl": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz",
"integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw=="
},
"node_modules/type": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/type/-/type-2.5.0.tgz",
"integrity": "sha512-180WMDQaIMm3+7hGXWf12GtdniDEy7nYcyFMKJn/eZz/6tSLXrUN9V0wKSbMjej0I1WHWbpREDEKHtqPQa9NNw=="
}
},
"dependencies": {
"applause-cli": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/applause-cli/-/applause-cli-1.7.0.tgz",
"integrity": "sha512-zO/nBR9x37Iqlm3R2Glo0NyDIevwbDPrISprK61Z/M01Xvm5dFkL2FZAONxxHctT9QoF71fiY22sFRuq8Sqxug=="
},
"d": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
"integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==",
"requires": {
"es5-ext": "^0.10.50",
"type": "^1.0.1"
},
"dependencies": {
"type": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz",
"integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg=="
}
}
},
"duration": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/duration/-/duration-0.2.2.tgz",
"integrity": "sha512-06kgtea+bGreF5eKYgI/36A6pLXggY7oR4p1pq4SmdFBn1ReOL5D8RhG64VrqfTTKNucqqtBAwEj8aB88mcqrg==",
"requires": {
"d": "1",
"es5-ext": "~0.10.46"
}
},
"es5-ext": {
"version": "0.10.53",
"resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz",
"integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==",
"requires": {
"es6-iterator": "~2.0.3",
"es6-symbol": "~3.1.3",
"next-tick": "~1.0.0"
}
},
"es6-iterator": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
"integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=",
"requires": {
"d": "1",
"es5-ext": "^0.10.35",
"es6-symbol": "^3.1.1"
}
},
"es6-symbol": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz",
"integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==",
"requires": {
"d": "^1.0.1",
"ext": "^1.1.2"
}
},
"event-emitter": {
"version": "0.3.5",
"resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
"integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=",
"requires": {
"d": "1",
"es5-ext": "~0.10.14"
}
},
"ext": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/ext/-/ext-1.6.0.tgz",
"integrity": "sha512-sdBImtzkq2HpkdRLtlLWDa6w4DX22ijZLKx8BMPUuKe1c5lbN6xwQDQCxSfxBQnHZ13ls/FH0MQZx/q/gr6FQg==",
"requires": {
"type": "^2.5.0"
}
},
"log": {
"version": "6.2.0",
"resolved": "https://registry.npmjs.org/log/-/log-6.2.0.tgz",
"integrity": "sha512-W1sDY5FqR6wlpygW8ZFSxCfBhKx/RzCHK5S+Br8zA14bAnwSgCm5hToIWzi0Yhy6x9Ppw7pyIV06r8F5cSRHUw==",
"requires": {
"d": "^1.0.1",
"duration": "^0.2.2",
"es5-ext": "^0.10.53",
"event-emitter": "^0.3.5",
"sprintf-kit": "^2.0.1",
"type": "^2.5.0"
}
},
"make-cert": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/make-cert/-/make-cert-1.2.1.tgz",
"integrity": "sha512-mvQkS2rSZE0rfgxycg/g2gcFaNm/XzDgkDF3CIp9StDaPU8f+3NL+gwg/qzsgL8bDy50jwak6OonVrAc/U/xEw==",
"requires": {
"node-forge": "^0.10.0"
}
},
"next-tick": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz",
"integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw="
},
"node-forge": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",
"integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA=="
},
"sprintf-kit": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/sprintf-kit/-/sprintf-kit-2.0.1.tgz",
"integrity": "sha512-2PNlcs3j5JflQKcg4wpdqpZ+AjhQJ2OZEo34NXDtlB0tIPG84xaaXhpA8XFacFiwjKA4m49UOYG83y3hbMn/gQ==",
"requires": {
"es5-ext": "^0.10.53"
}
},
"tweetnacl": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz",
"integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw=="
},
"type": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/type/-/type-2.5.0.tgz",
"integrity": "sha512-180WMDQaIMm3+7hGXWf12GtdniDEy7nYcyFMKJn/eZz/6tSLXrUN9V0wKSbMjej0I1WHWbpREDEKHtqPQa9NNw=="
}
}
}

25
package.json Normal file
View file

@ -0,0 +1,25 @@
{
"name": "systemquery",
"version": "1.0.0",
"description": "Distributed system information query tool",
"main": "src/index.mjs",
"scripts": {
"test": "echo \"No tests have been implemented yet\""
},
"repository": {
"type": "git",
"url": "git+https://github.com/sbrl/systemquery.git"
},
"author": "Starbeamrainbowlabs",
"license": "MPL-2.0",
"bugs": {
"url": "https://github.com/sbrl/systemquery/issues"
},
"homepage": "https://github.com/sbrl/systemquery#readme",
"dependencies": {
"applause-cli": "^1.7.0",
"log": "^6.2.0",
"make-cert": "^1.2.1",
"tweetnacl": "^1.0.3"
}
}

59
src/cli.mjs Normal file
View file

@ -0,0 +1,59 @@
"use strict";
import path from 'path';
import fs from 'fs';
import CliParser from 'applause-cli';
import l from 'log';
import ln from 'log-node'; ln();
import a from './lib/io/Ansi.mjs';
const __dirname = import.meta.url.slice(7, import.meta.url.lastIndexOf("/"));
async function load_subcommands(cli) {
let dirs = await fs.promises.readdir(path.join(__dirname, "subcommands"));
for(let dir of dirs) {
(await import(path.join(__dirname, "subcommands", dir, `meta.mjs`))).default(cli);
}
}
export default async function () {
let cli = new CliParser(path.resolve(__dirname, "../package.json"));
cli.argument("verbose", "Enable verbose debugging output", null, "boolean");
await load_subcommands(cli);
let settings_cli = cli.parse(process.argv.slice(2));
if(cli.current_subcommand == null)
cli.write_help_exit();
let subcommand_file = path.join(
__dirname,
"subcommands",
cli.current_subcommand,
`${cli.current_subcommand}.mjs`
);
if(!fs.existsSync(subcommand_file)) {
l.error(`Error: The subcommand '${cli.current_subcommand}' doesn't exist (try --help for a list of subcommands and their uses).`);
process.exit(1);
}
let settings = (await import("./settings.mjs")).default;
settings.cli = settings_cli;
try {
await (await import(subcommand_file)).default();
}
catch(error) {
console.error();
if(settings.cli.verbose)
throw error;
else
console.error(`${a.fred}${a.hicol}${error.message}${a.reset}`);
process.exit(1);
throw error;
}
}

8
src/index.mjs Executable file
View file

@ -0,0 +1,8 @@
#!/usr/bin/env node
"use strict";
import cli from './cli.mjs';
(async () => {
await cli();
})();

9
src/settings.mjs Normal file
View file

@ -0,0 +1,9 @@
"use strict";
const settings = {
// cli Settings from the CLI
};
export default settings;

View file

@ -0,0 +1,10 @@
"use strict";
import fs from 'fs';
import path from 'path';
import settings from '../../settings.mjs';
export default async function () {
// Do stuff
}

View file

@ -0,0 +1,13 @@
"use strict";
export default function(cli) {
cli.subcommand("download", "Downloads & anonymises Twitter data from the Academic API (not guaranteed to be downloaded in chronological order)")
.argument("search", "The search string to query the Twitter API with", null, "string")
.argument("start-time", "The start time to start downloading tweets from", null, "date")
.argument("end-time", "Optional. The end time to finish downloading tweets at", null, "date")
.argument("credentials", "Path to a (TOML formatted) credentials file to use. See the README for more information on the format.")
.argument("output", "Path to the directory to write the output to", null, "string")
.argument("max-query-length", "The maximum query length. Default: 1024. This only needs changing if you're not on the academic product track (which is the only track supported by this tool at the current time)", 1024, "integer")
.argument("tweets-per-request", "Max number of tweets to ask for per api call. Default: 100, can range from 10 to 500. Keep at a low value during testing, then bump it to max for the actual run.", 100, "integer")
.argument("no-replies", "Do not download replies to tweets. Speeds up execution, but of course yields less data.", false, "boolean");
}

View file

@ -0,0 +1,8 @@
"use strict";
export default function(cli) {
cli.subcommand("anonymise", "Anonymises tweets in a jsonl file. Output file is <filename_without_ext>-anonymised.<ext>. Note that this is done automatically on download - you do NOT need to run this manually unless you have data from elsewhere.")
.argument("input", "Path to the input file to anonymise.", null, "string")
.argument("type", "The type of object to anonymise. Default: tweet. Possible values: tweet, user.", "tweet", "string")
.argument("credentials", "Path to a (TOML formatted) credentials file that contains the salt to use for anonymisation. See the README for more information on the format.")
}

View file

@ -0,0 +1,5 @@
"use strict";
export default async function () {
// Do stuff
}