Implement tfrecodify subcommand.

It's all still untested, but that's the next step

rainfallwrangler/package-lock.json

@@ -11,6 +11,7 @@
       "dependencies": {
         "applause-cli": "^1.8.1",
         "gunzip-maybe": "^1.4.2",
+        "pretty-ms": "^7.0.1",
         "terrain50": "^1.10.1",
         "tfrecord-stream": "^0.2.0"
       }
@@ -274,6 +275,14 @@
       "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
       "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU="
     },
+    "node_modules/parse-ms": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz",
+      "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/peek-stream": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz",
@@ -284,6 +293,20 @@
         "through2": "^2.0.3"
       }
     },
+    "node_modules/pretty-ms": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz",
+      "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==",
+      "dependencies": {
+        "parse-ms": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/process-nextick-args": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@@ -698,6 +721,11 @@
       "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
       "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU="
     },
+    "parse-ms": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz",
+      "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA=="
+    },
     "peek-stream": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz",
@@ -708,6 +736,14 @@
         "through2": "^2.0.3"
       }
     },
+    "pretty-ms": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz",
+      "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==",
+      "requires": {
+        "parse-ms": "^2.1.0"
+      }
+    },
     "process-nextick-args": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",

rainfallwrangler/package.json

@@ -15,6 +15,7 @@
   "dependencies": {
     "applause-cli": "^1.8.1",
     "gunzip-maybe": "^1.4.2",
+    "pretty-ms": "^7.0.1",
     "terrain50": "^1.10.1",
     "tfrecord-stream": "^0.2.0"
   }

rainfallwrangler/src/lib/io/TFRecordWriter.mjs

@@ -4,6 +4,7 @@ import fs from 'fs';
 import path from 'path';
 
 import tfrecord from 'tfrecord-stream';
+import pretty_ms from 'pretty-ms';
 
 class TFRecordWriter {
 	constructor(dirpath, count_per_file) {
@@ -16,11 +17,11 @@ class TFRecordWriter {
 		this.#builder = tfrecord.createBuilder();
 	}
 	
-	write(reader_radar, reader_water) {
+	async write(reader_radar, reader_water) {
 		// TODO: Shuffle stuff about in the *Python* data pipeline
 		
 		let writer = null;
-		let i = -1, i_file = 0, count_this_file = 0;
+		let i = -1, i_file = 0, count_this_file = 0, time_start = new Date();
 		while(true) {
 			i++;
 			
@@ -45,6 +46,8 @@ class TFRecordWriter {
 			);
 			
 			await writer.writeExample(example_next);
+			
+			process.stderr.write(`Elapsed: ${pretty_ms(new Date() - time_start)}, Written ${count_this_file}/${i_file}/${i} examples/files/total\r`);
 		}
 		
 	}

rainfallwrangler/src/lib/io/Terrain50StreamReader.mjs

@@ -11,7 +11,7 @@ import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:terrain50str
 import array2d_classify_convert_bin from '../../manip/array2d_classify_convert_bin.mjs';
 
 class Terrain50StreamReader {	
-	constructor(threshold, tolerant = false) {
+	constructor(threshold = 0.1, tolerant = false) {
 		this.threshold = threshold;
 		
 		this.tolerant = tolerant;

rainfallwrangler/src/subcommands/tfrecordify/meta.mjs

@@ -10,5 +10,6 @@ export default function(cli) {
 				.map(el => parseInt(el))
 				.reverse();
 		})
-		.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer");
+		.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer")
+		.argument("output", "The path to the directory to write the generated TFRecord files to.", null, "string");
 }

rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs

@@ -1,6 +1,12 @@
 "use strict";
 
+import fs from 'fs';
+
 import settings from '../../settings.mjs';
+import TFRecordWriter from '../../lib/io/TFRecordWriter.mjs';
+import RadarReader from '../../lib/io/RadarReader.mjs';
+import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs';
+
 
 export default async function() {
 	if(typeof settings.water !== "string")
@@ -9,6 +15,15 @@ export default async function() {
 	if(typeof settings.rainfall !== "string")
 		throw new Error(`Error: No filepath to rainfall radar data specified.`);
 	
+	if(typeof settings.output !== "string")
+		throw new Error(`Error: No output directory specified.`);
 	
-	// TODO: Do the fanceh parsing stuff here
+	if(!fs.existsSync(settings.output))
+		await fs.promises.mkdir(settings.output, { recursive: true });
+		
+	const writer = new TFRecordWriter(settings.output, settings.count_file);
+	const reader_radar = new RadarReader();
+	const reader_water = new Terrain50StreamReader();
+	
+	await writer.write(reader_radar, reader_water);
 }