From 8a9cd6c1c0d5ef91ed006bb5d51f8a22e3bd273d Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Fri, 13 May 2022 19:06:15 +0100 Subject: [PATCH] Lay out some basic scaffolding I *really* hope this works. This is the 3rd major revision of this model. I've learnt a ton of stuff between now and my last attempt, so here's hoping that all goes well :D The basic idea behind this attempt is *Contrastive Learning*. If we don't get anything useful with this approach, then we can assume that it's not really possible / feasible. Something we need to watch out for is the variance (or rather lack thereof) in the dataset. We have 1.5M timesteps, but not a whole lot will be happening in most of those.... We may need to analyse the variance of the water depth data and extract a subsample that's more balanced. --- .gitignore | 341 +++++++ README.md | 26 + aimodel/src/__init__.py | 0 aimodel/src/index.py | 0 rainfallwrangler/package-lock.json | 865 ++++++++++++++++++ rainfallwrangler/package.json | 21 + rainfallwrangler/src/cli.mjs | 63 ++ rainfallwrangler/src/index.mjs | 12 + rainfallwrangler/src/lib/io/Ansi.mjs | 88 ++ rainfallwrangler/src/lib/io/Log.mjs | 71 ++ rainfallwrangler/src/lib/io/NamespacedLog.mjs | 44 + rainfallwrangler/src/settings.mjs | 6 + .../src/subcommands/tfrecordify/meta.mjs | 14 + .../subcommands/tfrecordify/tfrecordify.mjs | 14 + 14 files changed, 1565 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 aimodel/src/__init__.py create mode 100644 aimodel/src/index.py create mode 100644 rainfallwrangler/package-lock.json create mode 100644 rainfallwrangler/package.json create mode 100644 rainfallwrangler/src/cli.mjs create mode 100755 rainfallwrangler/src/index.mjs create mode 100644 rainfallwrangler/src/lib/io/Ansi.mjs create mode 100644 rainfallwrangler/src/lib/io/Log.mjs create mode 100644 rainfallwrangler/src/lib/io/NamespacedLog.mjs create mode 100644 rainfallwrangler/src/settings.mjs create mode 100644 rainfallwrangler/src/subcommands/tfrecordify/meta.mjs create mode 100644 rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f07fd77 --- /dev/null +++ b/.gitignore @@ -0,0 +1,341 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python,node,git,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=python,node,git,visualstudiocode + +### Git ### +# Created by git for backups. To disable backups in Git: +# $ git config --global mergetool.keepBackup false +*.orig + +# Created by git when using merge tools for conflicts +*.BACKUP.* +*.BASE.* +*.LOCAL.* +*.REMOTE.* +*_BACKUP_*.txt +*_BASE_*.txt +*_LOCAL_*.txt +*_REMOTE_*.txt + +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +### Node Patch ### +# Serverless Webpack directories +.webpack/ + +# Optional stylelint cache + +# SvelteKit build / generate output +.svelte-kit + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# Support for Project snippet scope +.vscode/*.code-snippets + +# Ignore code-workspaces +*.code-workspace + +# End of https://www.toptal.com/developers/gitignore/api/python,node,git,visualstudiocode diff --git a/README.md b/README.md new file mode 100644 index 0000000..945e779 --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +# Rainfall Radar + +> A model to predict water depth data from rainfall radar information. + +This is the 3rd major version of this model. + + + + + +## rainfallwrangler +`rainfallwrangler` is a Node.js application to wrangle the dataset into something more appropriate for training an AI efficiently. The rainfall radar and water depth data are considered temporally to be regular time steps. Here's a diagram explaining the terminology: + +``` +NOW +│ │ │Water depth +│▼ Rainfall Radar Data ▼│[Offset] │▼ +├─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┼─┬─┬─┬─┬─┼─┐ +│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ +│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ +│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ +│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ +└─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┴─┴─┴─┴─┴─┘ + │ +◄────────── Timesteps ─────────────► +``` diff --git a/aimodel/src/__init__.py b/aimodel/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimodel/src/index.py b/aimodel/src/index.py new file mode 100644 index 0000000..e69de29 diff --git a/rainfallwrangler/package-lock.json b/rainfallwrangler/package-lock.json new file mode 100644 index 0000000..a1cb47b --- /dev/null +++ b/rainfallwrangler/package-lock.json @@ -0,0 +1,865 @@ +{ + "name": "rainfallwrangler", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "rainfallwrangler", + "version": "1.0.0", + "license": "GPL-3.0", + "dependencies": { + "applause-cli": "^1.8.1", + "gunzip-maybe": "^1.4.2", + "terrain50": "^1.10.1", + "tfrecord-stream": "^0.2.0" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" + }, + "node_modules/@types/node": { + "version": "17.0.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.33.tgz", + "integrity": "sha512-miWq2m2FiQZmaHfdZNcbpp9PuXg34W5JZ5CrJ/BaS70VuhoJENBEQybeiYSaPBRNq6KQGnjfEnc/F3PN++D+XQ==" + }, + "node_modules/applause-cli": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/applause-cli/-/applause-cli-1.8.1.tgz", + "integrity": "sha512-ayN8TdWe69sDXRDNEKSUx7pO71XvVXd+v/bRWRCAbYwt2W82boA0NWQ/khfWJz4sXIWRY4tQIM2dsdYxj2s6ag==" + }, + "node_modules/awaitify-stream": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/awaitify-stream/-/awaitify-stream-1.0.2.tgz", + "integrity": "sha512-JE6mrRIPxhBQWt9Mu4u2XJF9V9xp4p5+Uxif/Ol/s2TFC/9+offUE50j/KQdaVWwMyGlKrlsE7Ncq/yhc6AJ+w==" + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "optional": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/browserify-zlib": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.1.4.tgz", + "integrity": "sha1-uzX4pRn2AOD6a4SFJByXnQFB+y0=", + "dependencies": { + "pako": "~0.2.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "node_modules/duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "dependencies": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/fast-crc32c": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-crc32c/-/fast-crc32c-2.0.0.tgz", + "integrity": "sha512-LIREwygxtxzHF11oLJ4xIVKu/ZWNgrj/QaGvaSD8ZggIsgCyCtSYevlrpWVqNau57ZwezV8K1HFBSjQ7FcRbTQ==", + "optionalDependencies": { + "sse4_crc32": "^6.0.1" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "optional": true + }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + }, + "node_modules/gunzip-maybe": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", + "integrity": "sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==", + "dependencies": { + "browserify-zlib": "^0.1.4", + "is-deflate": "^1.0.0", + "is-gzip": "^1.0.0", + "peek-stream": "^1.1.0", + "pumpify": "^1.3.3", + "through2": "^2.0.3" + }, + "bin": { + "gunzip-maybe": "bin.js" + } + }, + "node_modules/iconv-lite": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz", + "integrity": "sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/is-deflate": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-deflate/-/is-deflate-1.0.0.tgz", + "integrity": "sha1-yGKQHDwWH7CdrHzcfnhPgOmPLxQ=" + }, + "node_modules/is-gzip": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-gzip/-/is-gzip-1.0.0.tgz", + "integrity": "sha1-bKiwe5nHeZgCWQDlVc7Y7YCHmoM=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, + "node_modules/mgrs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", + "integrity": "sha1-+5FYjnjJACVnI5XLQLJffNatGCk=" + }, + "node_modules/nexline": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/nexline/-/nexline-1.2.2.tgz", + "integrity": "sha512-YLX5uoqNP7XVsXk889i8ZQcuMkukA4My4JD9wqTRLT+4dFo6QEEn+hU26J5H89m+mzW9BfhDgriGdbMEP06eeQ==", + "dependencies": { + "fs-extra": "^8.1.0", + "iconv-lite": "^0.5.0" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/nnng": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/nnng/-/nnng-1.0.0.tgz", + "integrity": "sha1-796D3fTAKtwDAQJbTrUcuzRlY0U=", + "dependencies": { + "proj4": "^2.3.10" + } + }, + "node_modules/node-addon-api": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-1.7.2.tgz", + "integrity": "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==", + "optional": true + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/pako": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", + "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=" + }, + "node_modules/peek-stream": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", + "integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==", + "dependencies": { + "buffer-from": "^1.0.0", + "duplexify": "^3.5.0", + "through2": "^2.0.3" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/proj4": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.8.0.tgz", + "integrity": "sha512-baC+YcD4xsSqJ+CpCZljj2gcQDhlKb+J+Zjv/2KSBwWNjk4M0pafgQsE+mWurd84tflMIsP+7j7mtIpFDHzQfQ==", + "dependencies": { + "mgrs": "1.0.0", + "wkt-parser": "^1.3.1" + } + }, + "node_modules/protobufjs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.2.tgz", + "integrity": "sha512-4BQJoPooKJl2G9j3XftkIXjoC9C0Av2NOrWmbLWT1vH32GcSUHjM0Arra6UfTsVyfMAuFzaLucXn1sadxJydAw==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + } + }, + "node_modules/pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/pumpify": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "dependencies": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" + } + }, + "node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/sse4_crc32": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/sse4_crc32/-/sse4_crc32-6.0.1.tgz", + "integrity": "sha512-FUTYXpLroqytNKWIfHzlDWoy9E4tmBB/RklNMy6w3VJs+/XEYAHgbiylg4SS43iOk/9bM0BlJ2EDpFAGT66IoQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "bindings": "^1.3.0", + "node-addon-api": "^1.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/stream-shift": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/terrain50": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/terrain50/-/terrain50-1.10.1.tgz", + "integrity": "sha512-zOOTRAAPvFKlbfII5fG0YiWHy/12x8Ms9kyG8RMs64rTy1tLzIhe3OwPeveONY6hP9LupG5DtCr9McP3n4wwww==", + "dependencies": { + "nexline": "^1.2.2", + "nnng": "^1.0.0" + } + }, + "node_modules/tfrecord-stream": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/tfrecord-stream/-/tfrecord-stream-0.2.0.tgz", + "integrity": "sha512-B7TtEQUqR6u/0drbPZBUUaBFVp4QbEPBgSiUolSarKuyIYqM6AAgB/Tre5gECoSacEl39hgHBnXm+xjj9zzIcA==", + "dependencies": { + "awaitify-stream": "^1.0.2", + "fast-crc32c": "^2.0.0", + "protobufjs": "^6.8.4" + } + }, + "node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "node_modules/wkt-parser": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.3.2.tgz", + "integrity": "sha512-A26BOOo7sHAagyxG7iuRhnKMO7Q3mEOiOT4oGUmohtN/Li5wameeU4S6f8vWw6NADTVKljBs8bzA8JPQgSEMVQ==" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + } + }, + "dependencies": { + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + }, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + }, + "@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" + }, + "@types/node": { + "version": "17.0.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.33.tgz", + "integrity": "sha512-miWq2m2FiQZmaHfdZNcbpp9PuXg34W5JZ5CrJ/BaS70VuhoJENBEQybeiYSaPBRNq6KQGnjfEnc/F3PN++D+XQ==" + }, + "applause-cli": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/applause-cli/-/applause-cli-1.8.1.tgz", + "integrity": "sha512-ayN8TdWe69sDXRDNEKSUx7pO71XvVXd+v/bRWRCAbYwt2W82boA0NWQ/khfWJz4sXIWRY4tQIM2dsdYxj2s6ag==" + }, + "awaitify-stream": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/awaitify-stream/-/awaitify-stream-1.0.2.tgz", + "integrity": "sha512-JE6mrRIPxhBQWt9Mu4u2XJF9V9xp4p5+Uxif/Ol/s2TFC/9+offUE50j/KQdaVWwMyGlKrlsE7Ncq/yhc6AJ+w==" + }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, + "browserify-zlib": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.1.4.tgz", + "integrity": "sha1-uzX4pRn2AOD6a4SFJByXnQFB+y0=", + "requires": { + "pako": "~0.2.0" + } + }, + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "requires": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, + "fast-crc32c": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-crc32c/-/fast-crc32c-2.0.0.tgz", + "integrity": "sha512-LIREwygxtxzHF11oLJ4xIVKu/ZWNgrj/QaGvaSD8ZggIsgCyCtSYevlrpWVqNau57ZwezV8K1HFBSjQ7FcRbTQ==", + "requires": { + "sse4_crc32": "^6.0.1" + } + }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "optional": true + }, + "fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + }, + "gunzip-maybe": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", + "integrity": "sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==", + "requires": { + "browserify-zlib": "^0.1.4", + "is-deflate": "^1.0.0", + "is-gzip": "^1.0.0", + "peek-stream": "^1.1.0", + "pumpify": "^1.3.3", + "through2": "^2.0.3" + } + }, + "iconv-lite": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz", + "integrity": "sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-deflate": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-deflate/-/is-deflate-1.0.0.tgz", + "integrity": "sha1-yGKQHDwWH7CdrHzcfnhPgOmPLxQ=" + }, + "is-gzip": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-gzip/-/is-gzip-1.0.0.tgz", + "integrity": "sha1-bKiwe5nHeZgCWQDlVc7Y7YCHmoM=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, + "mgrs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", + "integrity": "sha1-+5FYjnjJACVnI5XLQLJffNatGCk=" + }, + "nexline": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/nexline/-/nexline-1.2.2.tgz", + "integrity": "sha512-YLX5uoqNP7XVsXk889i8ZQcuMkukA4My4JD9wqTRLT+4dFo6QEEn+hU26J5H89m+mzW9BfhDgriGdbMEP06eeQ==", + "requires": { + "fs-extra": "^8.1.0", + "iconv-lite": "^0.5.0" + } + }, + "nnng": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/nnng/-/nnng-1.0.0.tgz", + "integrity": "sha1-796D3fTAKtwDAQJbTrUcuzRlY0U=", + "requires": { + "proj4": "^2.3.10" + } + }, + "node-addon-api": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-1.7.2.tgz", + "integrity": "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==", + "optional": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "pako": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", + "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=" + }, + "peek-stream": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", + "integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==", + "requires": { + "buffer-from": "^1.0.0", + "duplexify": "^3.5.0", + "through2": "^2.0.3" + } + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "proj4": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.8.0.tgz", + "integrity": "sha512-baC+YcD4xsSqJ+CpCZljj2gcQDhlKb+J+Zjv/2KSBwWNjk4M0pafgQsE+mWurd84tflMIsP+7j7mtIpFDHzQfQ==", + "requires": { + "mgrs": "1.0.0", + "wkt-parser": "^1.3.1" + } + }, + "protobufjs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.2.tgz", + "integrity": "sha512-4BQJoPooKJl2G9j3XftkIXjoC9C0Av2NOrWmbLWT1vH32GcSUHjM0Arra6UfTsVyfMAuFzaLucXn1sadxJydAw==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + } + }, + "pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "pumpify": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "requires": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" + } + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sse4_crc32": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/sse4_crc32/-/sse4_crc32-6.0.1.tgz", + "integrity": "sha512-FUTYXpLroqytNKWIfHzlDWoy9E4tmBB/RklNMy6w3VJs+/XEYAHgbiylg4SS43iOk/9bM0BlJ2EDpFAGT66IoQ==", + "optional": true, + "requires": { + "bindings": "^1.3.0", + "node-addon-api": "^1.3.0" + } + }, + "stream-shift": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "terrain50": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/terrain50/-/terrain50-1.10.1.tgz", + "integrity": "sha512-zOOTRAAPvFKlbfII5fG0YiWHy/12x8Ms9kyG8RMs64rTy1tLzIhe3OwPeveONY6hP9LupG5DtCr9McP3n4wwww==", + "requires": { + "nexline": "^1.2.2", + "nnng": "^1.0.0" + } + }, + "tfrecord-stream": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/tfrecord-stream/-/tfrecord-stream-0.2.0.tgz", + "integrity": "sha512-B7TtEQUqR6u/0drbPZBUUaBFVp4QbEPBgSiUolSarKuyIYqM6AAgB/Tre5gECoSacEl39hgHBnXm+xjj9zzIcA==", + "requires": { + "awaitify-stream": "^1.0.2", + "fast-crc32c": "^2.0.0", + "protobufjs": "^6.8.4" + } + }, + "through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "requires": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "wkt-parser": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.3.2.tgz", + "integrity": "sha512-A26BOOo7sHAagyxG7iuRhnKMO7Q3mEOiOT4oGUmohtN/Li5wameeU4S6f8vWw6NADTVKljBs8bzA8JPQgSEMVQ==" + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } +} diff --git a/rainfallwrangler/package.json b/rainfallwrangler/package.json new file mode 100644 index 0000000..6de6150 --- /dev/null +++ b/rainfallwrangler/package.json @@ -0,0 +1,21 @@ +{ + "name": "rainfallwrangler", + "version": "1.0.0", + "description": "Wrangles rainfall radar and water depth data into something sensible.", + "main": "src/index.mjs", + "scripts": { + "test": "echo \"No tests have been implemented yet\"" + }, + "repository": { + "type": "git", + "url": "https://git.starbeamrainbowlabs.com/sbrl/PhD-Rainfall-Radar.git" + }, + "author": "Starbeamrainbowlabs", + "license": "GPL-3.0", + "dependencies": { + "applause-cli": "^1.8.1", + "gunzip-maybe": "^1.4.2", + "terrain50": "^1.10.1", + "tfrecord-stream": "^0.2.0" + } +} diff --git a/rainfallwrangler/src/cli.mjs b/rainfallwrangler/src/cli.mjs new file mode 100644 index 0000000..b761fcc --- /dev/null +++ b/rainfallwrangler/src/cli.mjs @@ -0,0 +1,63 @@ +"use strict"; + +import path from 'path'; +import fs from 'fs'; + +import CliParser from 'applause-cli'; + +import log from './lib/io/NamespacedLog.mjs'; const l = log("core"); +import { LOG_LEVELS } from './lib/io/Log.mjs'; +import a from './lib/io/Ansi.mjs'; + +import settings from './settings.mjs'; + +const __dirname = import.meta.url.slice(7, import.meta.url.lastIndexOf("/")); + +async function load_subcommands(cli) { + let dirs = await fs.promises.readdir(path.join(__dirname, "subcommands")); + for(let dir of dirs) { + (await import(path.join(__dirname, "subcommands", dir, `meta.mjs`))).default(cli); + } +} + +export default async function () { + let cli = new CliParser(path.resolve(__dirname, "../package.json")); + cli.argument("verbose", "Enable verbose debugging output", null, "boolean") + .argument("log-level", "Sets the log level. Value values: DEBUG, INFO (the default), LOG, WARN, ERROR, NONE", "INFO", "string"); + + await load_subcommands(cli); + + let cli_parsed = cli.parse(process.argv.slice(2)); + + if(cli.current_subcommand == null) + cli.write_help_exit(); + + Object.assign(settings, cli_parsed); + + l.level = LOG_LEVELS[settings.log_level]; + + let subcommand_file = path.join( + __dirname, + "subcommands", + cli.current_subcommand, + `${cli.current_subcommand}.mjs` + ); + + if(!fs.existsSync(subcommand_file)) { + l.error(`Error: The subcommand '${cli.current_subcommand}' doesn't exist (try --help for a list of subcommands and their uses).`); + process.exit(1); + } + + try { + await (await import(subcommand_file)).default(); + } + catch(error) { + console.error(); + if(settings.cli.verbose) + throw error; + else + console.error(`${a.fred}${a.hicol}${error.message}${a.reset}`); + process.exit(1); + throw error; + } +} diff --git a/rainfallwrangler/src/index.mjs b/rainfallwrangler/src/index.mjs new file mode 100755 index 0000000..832b3ce --- /dev/null +++ b/rainfallwrangler/src/index.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +"use strict"; + + +import cli from './cli.mjs'; + + +(async () => { + "use strict"; + + await cli(); +})(); diff --git a/rainfallwrangler/src/lib/io/Ansi.mjs b/rainfallwrangler/src/lib/io/Ansi.mjs new file mode 100644 index 0000000..26415ff --- /dev/null +++ b/rainfallwrangler/src/lib/io/Ansi.mjs @@ -0,0 +1,88 @@ +"use strict"; + +/** + * Generates various VT100 ANSI escape sequences. + * Ported from C#. + * @licence MPL-2.0 + * @source https://gist.github.com/a4edd3204a03f4eedb79785751efb0f3#file-ansi-cs + * @author Starbeamrainbowlabs + * GitHub: @sbrl | Twitter: @SBRLabs | Reddit: u/Starbeamrainbowlabs + ***** Changelog ***** + * 27th March 2019: + * - Initial public release + * 9th March 2020: + * - Add Italics (\u001b[3m]) + * - Export a new instance of it by default (makes it so that there's 1 global instance) + * 5th September 2020: + * - Add support for NO_COLOR environment variable + */ +class Ansi { + constructor() { + /** + * Whether we should *actually* emit ANSI escape codes or not. + * Useful when we want to output to a log file, for example + * @type {Boolean} + */ + this.enabled = true; + + this.escape_codes(); + } + + escape_codes() { + if(typeof process !== "undefined" && typeof process.env.NO_COLOR == "string") { + this.enabled = false; + return; + } + // Solution on how to output ANSI escape codes in C# from here: + // https://www.jerriepelser.com/blog/using-ansi-color-codes-in-net-console-apps + this.reset = this.enabled ? "\u001b[0m" : ""; + this.hicol = this.enabled ? "\u001b[1m" : ""; + this.locol = this.enabled ? "\u001b[2m" : ""; + this.italics = this.enabled ? "\u001b[3m" : ""; + this.underline = this.enabled ? "\u001b[4m" : ""; + this.inverse = this.enabled ? "\u001b[7m" : ""; + this.fblack = this.enabled ? "\u001b[30m" : ""; + this.fred = this.enabled ? "\u001b[31m" : ""; + this.fgreen = this.enabled ? "\u001b[32m" : ""; + this.fyellow = this.enabled ? "\u001b[33m" : ""; + this.fblue = this.enabled ? "\u001b[34m" : ""; + this.fmagenta = this.enabled ? "\u001b[35m" : ""; + this.fcyan = this.enabled ? "\u001b[36m" : ""; + this.fwhite = this.enabled ? "\u001b[37m" : ""; + this.bblack = this.enabled ? "\u001b[40m" : ""; + this.bred = this.enabled ? "\u001b[41m" : ""; + this.bgreen = this.enabled ? "\u001b[42m" : ""; + this.byellow = this.enabled ? "\u001b[43m" : ""; + this.bblue = this.enabled ? "\u001b[44m" : ""; + this.bmagenta = this.enabled ? "\u001b[45m" : ""; + this.bcyan = this.enabled ? "\u001b[46m" : ""; + this.bwhite = this.enabled ? "\u001b[47m" : ""; + } + + // Thanks to http://ascii-table.com/ansi-escape-sequences.php for the following ANSI escape sequences + up(lines = 1) { + return this.enabled ? `\u001b[${lines}A` : ""; + } + down(lines = 1) { + return this.enabled ? `\u001b[${lines}B` : ""; + } + right(lines = 1) { + return this.enabled ? `\u001b[${lines}C` : ""; + } + left(lines = 1) { + return this.enabled ? `\u001b[${lines}D` : ""; + } + + jump_to(x, y) { + return this.enabled ? `\u001b[${y};${x}H` : ""; + } + + cursor_pos_save() { + return this.enabled ? `\u001b[s` : ""; + } + cursor_pos_restore() { + return this.enabled ? `\u001b[u` : ""; + } +} + +export default (new Ansi()); diff --git a/rainfallwrangler/src/lib/io/Log.mjs b/rainfallwrangler/src/lib/io/Log.mjs new file mode 100644 index 0000000..65a48f3 --- /dev/null +++ b/rainfallwrangler/src/lib/io/Log.mjs @@ -0,0 +1,71 @@ +"use strict"; + +import a from './Ansi.mjs'; + +const LOG_LEVELS = { + DEBUG: 0, + INFO: 1, + LOG: 2, + WARN: 4, + ERROR: 8, + NONE: 2048 +}; + +class Log { + constructor() { + this.start = new Date(); + + this.level = LOG_LEVELS.DEBUG; + } + + + debug(...message) { + if(this.level > LOG_LEVELS.DEBUG) return; + this.__do_log("debug", ...message); + } + + info(...message) { + if(this.level > LOG_LEVELS.INFO) return; + this.__do_log("info", ...message); + } + + log(...message) { + if(this.level > LOG_LEVELS.LOG) return; + this.__do_log("log", ...message); + } + + warn(...message) { + if(this.level > LOG_LEVELS.WARN) return; + this.__do_log("warn", ...message); + } + + error(...message) { + if(this.level > LOG_LEVELS.ERROR) return; + this.__do_log("error", ...message); + } + + + __do_log(level, ...message) { + message.unshift(`${a.locol}[ ${((new Date() - this.start) / 1000).toFixed(3)}]${a.reset}`); + let part = `[ ${level} ]`; + switch(level) { + case "debug": + part = a.locol + part; + break; + case "warn": + part = a.fyellow + part; + break; + case "error": + part = a.fred + part; + break; + } + message.unshift(part) + + console.error(...message); + } +} + +// You won't normally need these +export { LOG_LEVELS }; + +export default new Log(); diff --git a/rainfallwrangler/src/lib/io/NamespacedLog.mjs b/rainfallwrangler/src/lib/io/NamespacedLog.mjs new file mode 100644 index 0000000..afd23cd --- /dev/null +++ b/rainfallwrangler/src/lib/io/NamespacedLog.mjs @@ -0,0 +1,44 @@ +"use strict"; + +import a from './Ansi.mjs'; +import l from './Log.mjs'; + + +function hash_numeric(str) { + let hash = 0; + for (let i = 0; i < str.length; i++) { + hash = ((hash << 5) - hash) + str.charCodeAt(i); + hash |= 0; // Convert to 32bit integer + } + return hash; +} + +class NamespacedLog { + constructor(namespace) { + this.namespace = namespace; + + // From https://github.com/debug-js/debug/blob/master/src/node.js#L35 + let colours = [ + 20, 21, 26, 27, 32, 33, 38, 39, 40, 41, 42, 43, 44, 45, 56, 57, 62, + 63, 68, 69, 74, 75, 76, 77, 78, 79, 80, 81, 92, 93, 98, 99, 112, + 113, 128, 129, 134, 135, 148, 149, 160, 161, 162, 163, 164, 165, + 166, 167, 168, 169, 170, 171, 172, 173, 178, 179, 184, 185, 196, + 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 214, 215, 220, 221 + ]; + this.nscolour = colours[Math.abs(hash_numeric(this.namespace)) % colours.length]; + let colouransi = `\u001b[3${(this.nscolour < 8 ? this.nscolour : '8;5;' + this.nscolour)}m`; + if(!a.enabled) colouransi = ""; + this.ns = `${colouransi}${this.namespace}${a.reset}`; + } + + debug(...msg) { l.debug(this.ns, ...msg); } + info(...msg) { l.info(this.ns, ...msg); } + log(...msg) { l.log(this.ns, ...msg); } + warn(...msg) { l.warn(this.ns, ...msg); } + error(...msg) { l.error(this.ns, ...msg); } +} + +export default function(namespace) { + return new NamespacedLog(namespace); +}; diff --git a/rainfallwrangler/src/settings.mjs b/rainfallwrangler/src/settings.mjs new file mode 100644 index 0000000..b79cf58 --- /dev/null +++ b/rainfallwrangler/src/settings.mjs @@ -0,0 +1,6 @@ +"use strict"; + +// An empty container object to store the settings in +export default { + +}; diff --git a/rainfallwrangler/src/subcommands/tfrecordify/meta.mjs b/rainfallwrangler/src/subcommands/tfrecordify/meta.mjs new file mode 100644 index 0000000..e13ca46 --- /dev/null +++ b/rainfallwrangler/src/subcommands/tfrecordify/meta.mjs @@ -0,0 +1,14 @@ +"use strict"; + +export default function(cli) { + cli.subcommand("tfrecordify", "Converts rianfall radar and water depth data to a directory of tfrecord files.") + .argument("water", "Path to the water depths file, formatted as a stream of terrain50 objects. May or may not be gzipped.", null, "string") + .argument("rainfall", "Path to the rainfall radar data, formatted as jsonl. May or may not be gzipped.", null, "string") + .argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number.", 64*64) + .argument("rainfall-pattern", "The pattern of the number of time steps to average, as a comma-separated list of numbers. Given a point in time, each successive number specified works BACKWARDS from that point. For example, 1,4,10 would be 3 channels: 1 time step on it's own, then average the next 4 time steps, then average the next 10 steps.", "1,3,3,5,12,24,48", function(value) { + return value.split(",") + .map(el => parseInt(el)) + .reverse(); + }) + .argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer"); +} diff --git a/rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs b/rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs new file mode 100644 index 0000000..0707b5d --- /dev/null +++ b/rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs @@ -0,0 +1,14 @@ +"use strict"; + +import settings from '../../settings.mjs'; + +export default async function() { + if(typeof settings.water !== "string") + throw new Error(`Error: No filepath to water depth data specified.`); + + if(typeof settings.rainfall !== "string") + throw new Error(`Error: No filepath to rainfall radar data specified.`); + + + // TODO: Do the fanceh parsing stuff here +}