recordify: fix process exiting and imcomplete files issues

• Node.js not exiting at all
 • Node.js exiting on end_safe ing stream.Writable (?????)
 • Incomplete files - "unexpected end of file" errors and invalid JSON

rainfallwrangler/src/lib/io/GzipChildProcess.mjs

@@ -0,0 +1,81 @@
+"use strict";
+
+import EventEmitter from 'events';
+import child_process from 'child_process';
+
+import log from './NamespacedLog.mjs'; const l = log("gzipchildprocess");
+import { end_safe } from './StreamHelpers.mjs';
+
+
+/**
+ * Spawns and manages a gzip child process.
+ * @deprecated Use spawn-stream instead
+ * @extends EventEmitter
+ */
+class GzipChildProcess extends EventEmitter {
+	get stdin() { return this.child_process.stdin; }
+	get stdout() { return this.child_process.stdout; }
+	get stderr() { return this.child_process.stderr; }
+	
+	constructor(auto_start = true) {
+		super();
+		
+		this.debug = false;
+		this.child_process = null;
+		
+		this.has_ended = false;
+		
+		if(auto_start)
+			this.start();
+	}
+	
+	start() {
+		if(this.child_process != null)
+			throw new Error("Invalid Operation: Can't start the child process, since it's already been started.");
+		
+		this.child_process = child_process.spawn(
+			"gzip", [], {
+				// Pipe stdin + stdout; send error to the parent process
+				stdio: [ "pipe", "pipe", "inherit" ]
+			}
+		);
+		this.child_process.on("close", () => {
+			if(this.debug) l.debug("Close event triggered");
+			this.has_ended = true;
+			this.emit("close");
+		});
+		// FUTURE: Perhaps just throwing the error would be a better choice?
+		this.child_process.on("error", (error) => {
+			this.emit("error", error);
+		});
+		
+	}
+	
+	/**
+	 * Returns a Promise that resolves when the gzip process gracefully exits.
+	 * If the gzip child process has already exited, then it resolves immediately.
+	 * Gracefully closes the process by closing stdin.
+	 * @return	{Promise}
+	 */
+	async close() {
+		if(this.debug) l.debug("end_gracefully called");
+		if(this.has_ended) {
+			if(this.debug) l.debug("It's been ended already - nothing to do");
+			return;
+		}
+		if(!this.stdin.writableFinished) {
+			if(this.debug) l.debug("Closing stdin");
+			await end_safe(this.stdin);
+			if(this.debug) l.debug("stdin closed successfully");
+		}
+		if(this.has_ended) {
+			if(this.debug) l.debug("It's been ended already - nothing to do");
+			return;
+		}
+		if(this.debug) l.debug("Waiting for close event");
+		await EventEmitter.once(this, "close");
+		if(this.debug) l.debug("Close event fired, our work is done");
+	}
+}
+
+export default GzipChildProcess;

rainfallwrangler/src/lib/io/RadarReader.mjs

@@ -140,8 +140,6 @@ class RadarReader {
 	}
 	
 	async close() {
-		if(this.stream_in !== null) this.stream_in.close();
-		if(this.stream_extractor !== null) await end_safe(this.stream_extractor);
 		if(this.reader !== null) this.reader.close();
 		
 		this.stream_in = null;

rainfallwrangler/src/lib/io/RecordWrangler.mjs

@@ -3,10 +3,11 @@
 import fs from 'fs';
 import path from 'path';
 
+import log from '../../lib/io/NamespacedLog.mjs'; const l = log("recordwrangler");
+
 import RecordBuilder from '../record/RecordBuilder.mjs';
 import RecordsWriter from '../record/RecordsWriter.mjs';
 import pretty_ms from 'pretty-ms';
-import terrain50_analyse_frequencies from 'terrain50/src/static/Terrain50AnalyseFrequencies.mjs';
 import { end_safe } from './StreamHelpers.mjs';
 
 class RecordWrangler {
@@ -43,7 +44,10 @@ class RecordWrangler {
 			
 			const sample_radar = await reader_radar.next();
 			const sample_water = await reader_water.next();
-			if(sample_radar.done || sample_water.done) break;
+			if(sample_radar.done || sample_water.done) {
+				l.log(`Done because ${sample_radar.done?"radar":"water"} reader is out of records`);
+				break;
+			}
 			
 			const example_next = this.make_example(
 				sample_radar.value,
@@ -61,7 +65,7 @@ class RecordWrangler {
 		}
 		await writer.close();
 		
-		console.log(`\nComplete! ${i_file}/${i} files/records_total written in ${pretty_ms(new Date() - time_start)}`);
+		console.log(`\nComplete! ${i_file}/${i} files/records_total written in ${pretty_ms(new Date() - time_start)}\n`);
 	}
 	
 	make_example(sample_radar, sample_water) {

rainfallwrangler/src/lib/io/StreamHelpers.mjs

@@ -42,7 +42,7 @@ function write_safe(stream_out, data) {
  */
 function end_safe(stream, chunk = undefined) {
 	return new Promise((resolve, _reject) => {
-		stream.once("finish", resolve);
+		stream.once("finish", () => { console.log(`end_safe DEBUG finish`); resolve(); });
 		if(typeof chunk == "undefined") stream.end();
 		else stream.end(chunk);
 	});

rainfallwrangler/src/lib/record/RecordsWriter.mjs

@@ -2,17 +2,18 @@
 
 import fs from 'fs';
 
-import ChildProcess from 'duplex-child-process';
+import log from '../../lib/io/NamespacedLog.mjs'; const l = log("recordswriter");
 
+import GzipChildProcess from '../io/GzipChildProcess.mjs';
 import { write_safe, end_safe } from '../io/StreamHelpers.mjs';
 
 class RecordsWriter {
 	#stream_out = null;
-	#gzip = ChildProcess.spawn("gzip");
+	#gzip = new GzipChildProcess();
 	
 	constructor(filepath) {
 		this.#stream_out = fs.createWriteStream(filepath);
-		this.#gzip.pipe(this.#stream_out);
+		this.#gzip.stdout.pipe(this.#stream_out);
 	}
 	
 	/**
@@ -22,7 +23,7 @@ class RecordsWriter {
 	 */
 	async write(sample) {
 		const str = JSON.stringify(Object.fromEntries(sample));
-		await write_safe(this.#gzip, str+"\n");
+		await write_safe(this.#gzip.stdin, str+"\n");
 	}
 	
 	/**
@@ -31,7 +32,7 @@ class RecordsWriter {
 	 * @return	{Promise}
 	 */
 	async write_raw(line) {
-		await write_safe(this.#gzip, line+"\n");
+		await write_safe(this.#gzip.stdin, line+"\n");
 	}
 	
 	/**
@@ -40,8 +41,8 @@ class RecordsWriter {
 	 * @return	{Promise}
 	 */
 	async close() {
-		await end_safe(this.#gzip);
-		await end_safe(this.#stream_out);
+		await this.#gzip.close();
+		// Closing this.#stream_out causes a silent crash O.o 2022-07-08 @sbrl Node.js 18.4.0
 	}
 }
 

rainfallwrangler/src/subcommands/recordify/recordify.mjs

@@ -7,7 +7,7 @@ import RecordWrangler from '../../lib/io/RecordWrangler.mjs';
 import RadarWrangler from '../../lib/RadarWrangler.mjs';
 import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs';
 
-import log from './NamespacedLog.mjs'; const l = log("recordify");
+import log from '../../lib/io/NamespacedLog.mjs'; const l = log("recordify");
 
 export default async function() {
 	if(typeof settings.water !== "string")
@@ -29,7 +29,7 @@ export default async function() {
 	
 	await writer.write(reader_radar.iterate(settings.rainfall), reader_water.iterate(settings.water));
 	
-	l.log("Closing reader reader")
+	l.log("Closing radar reader")
 	await reader_radar.close();
 	l.log("Closing water depth data reader")
 	await reader_water.close();