From 460e2bc1aaa3e22a12a6c46ca5537cd47ded2376 Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Mon, 28 Sep 2020 17:55:03 +0100 Subject: [PATCH] Add new analyse-frequencies subcommand --- Changelog.md | 6 +++ package-lock.json | 6 +-- package.json | 2 +- src/Helpers/StreamHelpers.mjs | 50 +++++++++++++++++++ src/Subcommands/analyse-frequencies/index.mjs | 50 +++++++++++++++++++ src/Subcommands/analyse-frequencies/meta.toml | 19 +++++++ 6 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 src/Helpers/StreamHelpers.mjs create mode 100644 src/Subcommands/analyse-frequencies/index.mjs create mode 100644 src/Subcommands/analyse-frequencies/meta.toml diff --git a/Changelog.md b/Changelog.md index f72ed22..62dd656 100644 --- a/Changelog.md +++ b/Changelog.md @@ -13,9 +13,15 @@ npm install --save terrain50-cli ----- +## v1.5 + - Update dependencies + - Add new `analyse-frequencies` subcommand + + ## v1.4.1 - `validate` subcommand in `stream` mode: Write aggregated stats when validation is complete + ## v1.4 - [BREAKING] Add new `--use-regex` flag to `validate` subcommand (only takes effect in stream mode: `--mode stream`) diff --git a/package-lock.json b/package-lock.json index 7ec79ae..f6e1587 100644 --- a/package-lock.json +++ b/package-lock.json @@ -338,9 +338,9 @@ } }, "terrain50": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/terrain50/-/terrain50-1.8.0.tgz", - "integrity": "sha512-G6wJw0ysZMet0KCqOZr9Cr0OWTjXn00BpZy0SVMfgIE/GiDTeFbCt2EeHANBwKwc93ItLxGua4nWI+5v42jHmw==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/terrain50/-/terrain50-1.8.1.tgz", + "integrity": "sha512-bPAOCOcDHgHoLf2Y7uvk09kDZBRKSkIjBNH5kLzufjzaimxaWVJiALegO9RYw5sHRlU+nzbPZ/0trdJjSs/CRQ==", "requires": { "nexline": "^1.2.1", "nnng": "^1.0.0" diff --git a/package.json b/package.json index 6e10c74..ea69a63 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "chroma-js": "^2.1.0", "image-encode": "^1.3.0", "nexline": "^1.2.1", - "terrain50": "^1.8.0" + "terrain50": "^1.8.1" }, "bin": { "terrain50": "./src/index.mjs" diff --git a/src/Helpers/StreamHelpers.mjs b/src/Helpers/StreamHelpers.mjs new file mode 100644 index 0000000..aa63cac --- /dev/null +++ b/src/Helpers/StreamHelpers.mjs @@ -0,0 +1,50 @@ +"use strict"; + +/** + * Writes data to a stream, automatically waiting for the drain event if asked. + * See also write_safe. + * @param {stream.Writable} stream_out The writable stream to write to. + * @param {string|Buffer|Uint8Array} data The data to write. + * @return {Promise} A promise that resolves when writing is complete. + * @private + */ +function write_safe(stream_out, data) { + return new Promise(function (resolve, reject) { + // Handle errors + let handler_error = (error) => { + stream_out.off("error", handler_error); + reject(error); + }; + stream_out.on("error", handler_error); + + if(typeof data == "string" ? stream_out.write(data, "utf-8") : stream_out.write(data)) { + // We're good to go + stream_out.off("error", handler_error); + resolve(); + } + else { + // We need to wait for the drain event before continuing + stream_out.once("drain", () => { + stream_out.off("error", handler_error); + resolve(); + }); + } + }); +} + +/** + * Waits for the given stream to end and finish writing data. + * NOTE: This function is not tested and guaranteed yet. (ref #10 the HydroIndexWriter bug) + * @param {stream.Writable} stream The stream to end. + * @param {Buffer|string} [chunk=undefined] Optional. A chunk to write when calling .end(). + * @return {Promise} A Promise that resolves when writing is complete. + * @private + */ +function end_safe(stream, chunk = undefined) { + return new Promise((resolve, _reject) => { + if(typeof chunk == "undefined") stream.end(resolve); + else stream.end(chunk, resolve); + }); +} + +export { write_safe, end_safe }; diff --git a/src/Subcommands/analyse-frequencies/index.mjs b/src/Subcommands/analyse-frequencies/index.mjs new file mode 100644 index 0000000..a0d7d02 --- /dev/null +++ b/src/Subcommands/analyse-frequencies/index.mjs @@ -0,0 +1,50 @@ +"use strict"; + +import fs from 'fs'; + +import Terrain50 from 'terrain50'; + +import a from '../../Helpers/Ansi.mjs'; +import l from '../../Helpers/Log.mjs'; + +import { percentage } from '../../Helpers/MathsHelpers.mjs'; +import { write_safe, end_safe } from '../../Helpers/StreamHelpers.mjs'; + +export default async function(settings) { + // 1: Parse settings + let stream_in = process.stdin; + if(settings.cli.input !== "-") { + l.log(`Reading from ${a.hicol}${settings.cli.input}${a.reset}`); + stream_in = fs.createReadStream(settings.cli.input); + } + else + l.log(`Reading from stdin`); + + let stream_out = process.stdout; + if(settings.cli.output !== "-") { + l.log(`Writing to ${a.hicol}${settings.cli.output}${a.reset}`); + stream_out = fs.createWriteStream(settings.cli.output); + } + else + l.log(`Writing to stdout`); + + // ------------------------------------------------------------------------ + + let result_map = await Terrain50.AnalyseFrequencies( + Terrain50.ParseStream(stream_in), + settings.cli.ignore_nodata + ); + + let result_arr = []; + for(const [ key, value ] of result_map) { + result_arr.push([ key, value ]); + } + result_arr.sort((a, b) => a[0] - b[0]); + + for(const [ key, value ] of result_arr) { + await write_safe(stream_out, `${key} ${value}\n`); + } + await end_safe(stream_out); + + stream_in.destroy(); +} diff --git a/src/Subcommands/analyse-frequencies/meta.toml b/src/Subcommands/analyse-frequencies/meta.toml new file mode 100644 index 0000000..02dcbee --- /dev/null +++ b/src/Subcommands/analyse-frequencies/meta.toml @@ -0,0 +1,19 @@ +description = "Count the frequencies of data values. Data values are rounded down before counting." + +[[arguments]] +name = "input" +description = "The input file to analyse (default: stdin)" +default_value = "-" +type = "string" + +[[arguments]] +name = "output" +description = "The output file to write the results to as tab-separated-values (default: stdout)" +default_value = "-" +type = "string" + +[[arguments]] +name = "ignore-nodata" +description = "Ignore NODATA values when analysing." +default_value = false +type = "boolean"