From cc5efbae8a731c8e2c722cf93600f80518b02680 Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Thu, 19 May 2022 17:15:15 +0100 Subject: [PATCH] Implement tfrecodify subcommand. It's all still untested, but that's the next step --- rainfallwrangler/package-lock.json | 36 +++++++++++++++++++ rainfallwrangler/package.json | 1 + .../src/lib/io/TFRecordWriter.mjs | 7 ++-- .../src/lib/io/Terrain50StreamReader.mjs | 2 +- .../src/subcommands/tfrecordify/meta.mjs | 3 +- .../subcommands/tfrecordify/tfrecordify.mjs | 17 ++++++++- 6 files changed, 61 insertions(+), 5 deletions(-) diff --git a/rainfallwrangler/package-lock.json b/rainfallwrangler/package-lock.json index 90aae1c..6ae5569 100644 --- a/rainfallwrangler/package-lock.json +++ b/rainfallwrangler/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "applause-cli": "^1.8.1", "gunzip-maybe": "^1.4.2", + "pretty-ms": "^7.0.1", "terrain50": "^1.10.1", "tfrecord-stream": "^0.2.0" } @@ -274,6 +275,14 @@ "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=" }, + "node_modules/parse-ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==", + "engines": { + "node": ">=6" + } + }, "node_modules/peek-stream": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", @@ -284,6 +293,20 @@ "through2": "^2.0.3" } }, + "node_modules/pretty-ms": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", + "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", + "dependencies": { + "parse-ms": "^2.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -698,6 +721,11 @@ "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=" }, + "parse-ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" + }, "peek-stream": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", @@ -708,6 +736,14 @@ "through2": "^2.0.3" } }, + "pretty-ms": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", + "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", + "requires": { + "parse-ms": "^2.1.0" + } + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", diff --git a/rainfallwrangler/package.json b/rainfallwrangler/package.json index fba719d..68ca817 100644 --- a/rainfallwrangler/package.json +++ b/rainfallwrangler/package.json @@ -15,6 +15,7 @@ "dependencies": { "applause-cli": "^1.8.1", "gunzip-maybe": "^1.4.2", + "pretty-ms": "^7.0.1", "terrain50": "^1.10.1", "tfrecord-stream": "^0.2.0" } diff --git a/rainfallwrangler/src/lib/io/TFRecordWriter.mjs b/rainfallwrangler/src/lib/io/TFRecordWriter.mjs index 932366d..3d81a82 100644 --- a/rainfallwrangler/src/lib/io/TFRecordWriter.mjs +++ b/rainfallwrangler/src/lib/io/TFRecordWriter.mjs @@ -4,6 +4,7 @@ import fs from 'fs'; import path from 'path'; import tfrecord from 'tfrecord-stream'; +import pretty_ms from 'pretty-ms'; class TFRecordWriter { constructor(dirpath, count_per_file) { @@ -16,11 +17,11 @@ class TFRecordWriter { this.#builder = tfrecord.createBuilder(); } - write(reader_radar, reader_water) { + async write(reader_radar, reader_water) { // TODO: Shuffle stuff about in the *Python* data pipeline let writer = null; - let i = -1, i_file = 0, count_this_file = 0; + let i = -1, i_file = 0, count_this_file = 0, time_start = new Date(); while(true) { i++; @@ -45,6 +46,8 @@ class TFRecordWriter { ); await writer.writeExample(example_next); + + process.stderr.write(`Elapsed: ${pretty_ms(new Date() - time_start)}, Written ${count_this_file}/${i_file}/${i} examples/files/total\r`); } } diff --git a/rainfallwrangler/src/lib/io/Terrain50StreamReader.mjs b/rainfallwrangler/src/lib/io/Terrain50StreamReader.mjs index 16114a8..5d3d12a 100644 --- a/rainfallwrangler/src/lib/io/Terrain50StreamReader.mjs +++ b/rainfallwrangler/src/lib/io/Terrain50StreamReader.mjs @@ -11,7 +11,7 @@ import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:terrain50str import array2d_classify_convert_bin from '../../manip/array2d_classify_convert_bin.mjs'; class Terrain50StreamReader { - constructor(threshold, tolerant = false) { + constructor(threshold = 0.1, tolerant = false) { this.threshold = threshold; this.tolerant = tolerant; diff --git a/rainfallwrangler/src/subcommands/tfrecordify/meta.mjs b/rainfallwrangler/src/subcommands/tfrecordify/meta.mjs index e13ca46..ba65413 100644 --- a/rainfallwrangler/src/subcommands/tfrecordify/meta.mjs +++ b/rainfallwrangler/src/subcommands/tfrecordify/meta.mjs @@ -10,5 +10,6 @@ export default function(cli) { .map(el => parseInt(el)) .reverse(); }) - .argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer"); + .argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer") + .argument("output", "The path to the directory to write the generated TFRecord files to.", null, "string"); } diff --git a/rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs b/rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs index 0707b5d..60c4397 100644 --- a/rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs +++ b/rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs @@ -1,6 +1,12 @@ "use strict"; +import fs from 'fs'; + import settings from '../../settings.mjs'; +import TFRecordWriter from '../../lib/io/TFRecordWriter.mjs'; +import RadarReader from '../../lib/io/RadarReader.mjs'; +import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs'; + export default async function() { if(typeof settings.water !== "string") @@ -9,6 +15,15 @@ export default async function() { if(typeof settings.rainfall !== "string") throw new Error(`Error: No filepath to rainfall radar data specified.`); + if(typeof settings.output !== "string") + throw new Error(`Error: No output directory specified.`); - // TODO: Do the fanceh parsing stuff here + if(!fs.existsSync(settings.output)) + await fs.promises.mkdir(settings.output, { recursive: true }); + + const writer = new TFRecordWriter(settings.output, settings.count_file); + const reader_radar = new RadarReader(); + const reader_water = new Terrain50StreamReader(); + + await writer.write(reader_radar, reader_water); }