Fix many bugs

Many bugs remain though

rainfallwrangler/package.json

@@ -13,6 +13,7 @@
   "author": "Starbeamrainbowlabs",
   "license": "GPL-3.0-or-later",
   "dependencies": {
+    "@tensorflow/tfjs-node-gpu": "^3.17.0",
     "applause-cli": "^1.8.1",
     "gunzip-maybe": "^1.4.2",
     "pretty-ms": "^7.0.1",

rainfallwrangler/src/cli.mjs

@@ -22,7 +22,7 @@ async function load_subcommands(cli) {
 
 export default async function () {
 	let cli = new CliParser(path.resolve(__dirname, "../package.json"));
-	cli.argument("verbose", "Enable verbose debugging output", null, "boolean")
+	cli.argument("verbose", "Enable verbose debugging output", false, "boolean")
 		.argument("log-level", "Sets the log level. Value values: DEBUG, INFO (the default), LOG, WARN, ERROR, NONE", "INFO", "string");
 	
 	await load_subcommands(cli);
@@ -53,7 +53,7 @@ export default async function () {
 	}
 	catch(error) {
 		console.error();
-		if(settings.cli.verbose)
+		if(settings.verbose)
 			throw error;
 		else
 			console.error(`${a.fred}${a.hicol}${error.message}${a.reset}`);

rainfallwrangler/src/lib/RadarWrangler.mjs

@@ -1,5 +1,9 @@
 "use strict";
 
+import RadarReader from './io/RadarReader.mjs';
+
+import array2d_pool from './manip/array2d_pool.mjs';
+
 class RadarWrangler {
 	/**
 	 * The total number of timesteps we need in the buffer before we can even consider generating a sample.

rainfallwrangler/src/lib/io/RadarReader.mjs

@@ -6,7 +6,7 @@ import { Readable } from 'stream';
 import nexline from 'nexline';
 import gunzip from 'gunzip-maybe';
 
-import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:radar");
+import log from './NamespacedLog.mjs'; const l = log("reader:radar");
 import interpolate from '../manip/array2d_interpolate.mjs';
 import transpose from '../manip/array2d_transpose.mjs';
 

rainfallwrangler/src/lib/io/TFRecordWriter.mjs

@@ -7,14 +7,14 @@ import tfrecord from 'tfrecord-stream';
 import pretty_ms from 'pretty-ms';
 
 class TFRecordWriter {
+	#builder = tfrecord.createBuilder();
+	
 	constructor(dirpath, count_per_file) {
 		this.dirpath = dirpath;
 		this.count_per_file = count_per_file;
 		
-		if(!fs.existsSync(dirpath))
-			fs.mkdirSync(dirpath);
-		
-		this.#builder = tfrecord.createBuilder();
+		if(!fs.existsSync(this.dirpath))
+			fs.mkdirSync(this.dirpath);
 	}
 	
 	async write(reader_radar, reader_water) {
@@ -28,7 +28,7 @@ class TFRecordWriter {
 			// Start writing to a new file when necessary
 			if(writer == null || count_this_file > this.count_per_file) {
 				if(writer !== null) await writer.close();
-				const filepath_next = path.join(dirpath, `${i_file}.tfrecord`);
+				const filepath_next = path.join(this.dirpath, `${i_file}.tfrecord`);
 				writer = await tfrecord.Writer.createFromStream(
 					fs.createWriteStream(filepath_next)
 				);

rainfallwrangler/src/lib/io/Terrain50StreamReader.mjs

@@ -6,9 +6,9 @@ import fs from 'fs';
 import path from 'path';
 
 
-import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:terrain50stream");
+import log from './NamespacedLog.mjs'; const l = log("reader:terrain50stream");
 
-import array2d_classify_convert_bin from '../../manip/array2d_classify_convert_bin.mjs';
+import array2d_classify_convert_bin from '../manip/array2d_classify_convert_bin.mjs';
 
 class Terrain50StreamReader {	
 	constructor(threshold = 0.1, tolerant = false) {

rainfallwrangler/src/subcommands/tfrecordify/meta.mjs

@@ -4,12 +4,12 @@ export default function(cli) {
 	cli.subcommand("tfrecordify", "Converts rianfall radar and water depth data to a directory of tfrecord files.")
 		.argument("water", "Path to the water depths file, formatted as a stream of terrain50 objects. May or may not be gzipped.", null, "string")
 		.argument("rainfall", "Path to the rainfall radar data, formatted as jsonl. May or may not be gzipped.",  null, "string")
-		.argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number.", 64*64)
-		.argument("rainfall-pattern", "The pattern of the number of time steps to average, as a comma-separated list of numbers. Given a point in time, each successive number specified works BACKWARDS from that point. For example, 1,4,10 would be 3 channels: 1 time step on it's own, then average the next 4 time steps, then average the next 10 steps.", "1,3,3,5,12,24,48", function(value) {
+		.argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number (default: 4096).", 64*64)
+		.argument("rainfall-pattern", "The pattern of the number of time steps to average, as a comma-separated list of numbers. Given a point in time, each successive number specified works BACKWARDS from that point. For example, 1,4,10 would be 3 channels: 1 time step on it's own, then average the next 4 time steps, then average the next 10 steps (default: 1,3,3,5,12,24,48).", [1,3,3,5,12,24,48], function(value) {
 			return value.split(",")
 				.map(el => parseInt(el))
 				.reverse();
 		})
-		.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer")
+		.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data. (default: 1)", 1, "integer")
 		.argument("output", "The path to the directory to write the generated TFRecord files to.", null, "string");
 }

rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs

@@ -4,7 +4,7 @@ import fs from 'fs';
 
 import settings from '../../settings.mjs';
 import TFRecordWriter from '../../lib/io/TFRecordWriter.mjs';
-import RadarReader from '../../lib/io/RadarReader.mjs';
+import RadarWrangler from '../../lib/RadarWrangler.mjs';
 import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs';
 
 
@@ -21,9 +21,10 @@ export default async function() {
 	if(!fs.existsSync(settings.output))
 		await fs.promises.mkdir(settings.output, { recursive: true });
 	
+	console.log("DEBUG", settings);
 	const writer = new TFRecordWriter(settings.output, settings.count_file);
-	const reader_radar = new RadarReader();
+	const reader_radar = new RadarWrangler(settings.rainfall_pattern);
 	const reader_water = new Terrain50StreamReader();
 	
-	await writer.write(reader_radar, reader_water);
+	await writer.write(reader_radar.iterate(settings.rainfall), reader_water.iterate(settings.water));
 }