Use GzipChildStream again

....spawn-stream appears to be causing issues in the new nimrod data 
downloader
This commit is contained in:
Starbeamrainbowlabs 2020-10-05 18:58:11 +01:00
parent aca3fb0278
commit d1b6e9e641
Signed by: sbrl
GPG Key ID: 1BE5172E637709C2
4 changed files with 73 additions and 10 deletions

5
package-lock.json generated
View File

@ -328,11 +328,6 @@
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
},
"spawn-stream": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/spawn-stream/-/spawn-stream-1.0.2.tgz",
"integrity": "sha1-W9VkENXISo7N6y5jEnEWytkQUMs="
},
"string-to-arraybuffer": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-to-arraybuffer/-/string-to-arraybuffer-1.0.2.tgz",

View File

@ -37,7 +37,6 @@
"chroma-js": "^2.1.0",
"image-encode": "^1.3.0",
"nexline": "^1.2.1",
"spawn-stream": "^1.0.2",
"terrain50": "^1.8.2"
},
"bin": {

View File

@ -0,0 +1,61 @@
"use strict";
import EventEmitter from 'events';
import child_process from 'child_process';
import { end_safe } from '../../helpers/StreamHelpers.mjs';
/**
* Spawns and manages a gzip child process.
* @deprecated Use spawn-stream instead
* @extends EventEmitter
*/
class GzipChildProcess extends EventEmitter {
get stdin() { return this.child_process.stdin; }
get stdout() { return this.child_process.stdout; }
get stderr() { return this.child_process.stderr; }
constructor(auto_start = true) {
super();
this.child_process = null;
this.has_exited = false;
if(auto_start)
this.start();
}
start() {
if(this.child_process != null)
throw new Error("Invalid Operation: Can't start the child process, since it's already been started.");
this.child_process = child_process.spawn(
"gzip", [], {
// Pipe stdin + stdout; send error to the parent process
stdio: [ "pipe", "pipe", "inherit" ]
}
);
this.child_process.on("exit", () => {
this.has_exited = true;
this.emit("exit");
});
// FUTURE: Perhaps just throwing the error would be a better choice?
this.child_process.on("error", (error) => {
this.emit("error", error);
});
}
/**
* Returns a Promise that resolves when the gzip process exits.
* If the gzip child process has already exited, then it resolves immediately.
* @return {Promise}
*/
async end_gracefully() {
await end_safe(this.stdin);
if(this.has_exited) return;
await EventEmitter.once(this, "exit");
}
}
export default GzipChildProcess;

View File

@ -9,6 +9,9 @@ import SpawnStream from 'spawn-stream';
import a from '../../Helpers/Ansi.mjs';
import l from '../../Helpers/Log.mjs';
import GzipChildProcess from '../../Helpers/GzipChildProcess.mjs';
import { end_safe } from '../../Helpers/StreamHelpers.mjs';
export default async function(settings) {
// 1: Parse settings
let stream_in = process.stdin;
@ -39,15 +42,20 @@ export default async function(settings) {
// Create the output stream
let stream_out = fs.createWriteStream(output_filename),
stream_gzip = null;
gzip = null;
if(!settings.cli.no_gzip) {
stream_gzip = SpawnStream("gzip");
stream_gzip.pipe(stream_out);
gzip = new GzipChildProcess();
gzip.stdout.pipe(stream_out);
}
let init_stream = stream_out;
if(gzip !== null) init_stream = gzip.stdin;
// Write it to the output
await next.serialise(stream_gzip || stream_out, true);
await next.serialise(init_stream || stream_out, false);
if(gzip !== null) await gzip.end_gracefully();
await end_safe(stream_out);
// Update the user
l.log(`Written ${i+1} objects so far`);