Implement tfrecodify subcommand.

It's all still untested, but that's the next step
This commit is contained in:
Starbeamrainbowlabs 2022-05-19 17:15:15 +01:00
parent 0fa7ae9d6a
commit cc5efbae8a
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
6 changed files with 61 additions and 5 deletions

View file

@ -11,6 +11,7 @@
"dependencies": { "dependencies": {
"applause-cli": "^1.8.1", "applause-cli": "^1.8.1",
"gunzip-maybe": "^1.4.2", "gunzip-maybe": "^1.4.2",
"pretty-ms": "^7.0.1",
"terrain50": "^1.10.1", "terrain50": "^1.10.1",
"tfrecord-stream": "^0.2.0" "tfrecord-stream": "^0.2.0"
} }
@ -274,6 +275,14 @@
"resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
"integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=" "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU="
}, },
"node_modules/parse-ms": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz",
"integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==",
"engines": {
"node": ">=6"
}
},
"node_modules/peek-stream": { "node_modules/peek-stream": {
"version": "1.1.3", "version": "1.1.3",
"resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz",
@ -284,6 +293,20 @@
"through2": "^2.0.3" "through2": "^2.0.3"
} }
}, },
"node_modules/pretty-ms": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz",
"integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==",
"dependencies": {
"parse-ms": "^2.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/process-nextick-args": { "node_modules/process-nextick-args": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@ -698,6 +721,11 @@
"resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
"integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=" "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU="
}, },
"parse-ms": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz",
"integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA=="
},
"peek-stream": { "peek-stream": {
"version": "1.1.3", "version": "1.1.3",
"resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz",
@ -708,6 +736,14 @@
"through2": "^2.0.3" "through2": "^2.0.3"
} }
}, },
"pretty-ms": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz",
"integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==",
"requires": {
"parse-ms": "^2.1.0"
}
},
"process-nextick-args": { "process-nextick-args": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",

View file

@ -15,6 +15,7 @@
"dependencies": { "dependencies": {
"applause-cli": "^1.8.1", "applause-cli": "^1.8.1",
"gunzip-maybe": "^1.4.2", "gunzip-maybe": "^1.4.2",
"pretty-ms": "^7.0.1",
"terrain50": "^1.10.1", "terrain50": "^1.10.1",
"tfrecord-stream": "^0.2.0" "tfrecord-stream": "^0.2.0"
} }

View file

@ -4,6 +4,7 @@ import fs from 'fs';
import path from 'path'; import path from 'path';
import tfrecord from 'tfrecord-stream'; import tfrecord from 'tfrecord-stream';
import pretty_ms from 'pretty-ms';
class TFRecordWriter { class TFRecordWriter {
constructor(dirpath, count_per_file) { constructor(dirpath, count_per_file) {
@ -16,11 +17,11 @@ class TFRecordWriter {
this.#builder = tfrecord.createBuilder(); this.#builder = tfrecord.createBuilder();
} }
write(reader_radar, reader_water) { async write(reader_radar, reader_water) {
// TODO: Shuffle stuff about in the *Python* data pipeline // TODO: Shuffle stuff about in the *Python* data pipeline
let writer = null; let writer = null;
let i = -1, i_file = 0, count_this_file = 0; let i = -1, i_file = 0, count_this_file = 0, time_start = new Date();
while(true) { while(true) {
i++; i++;
@ -45,6 +46,8 @@ class TFRecordWriter {
); );
await writer.writeExample(example_next); await writer.writeExample(example_next);
process.stderr.write(`Elapsed: ${pretty_ms(new Date() - time_start)}, Written ${count_this_file}/${i_file}/${i} examples/files/total\r`);
} }
} }

View file

@ -11,7 +11,7 @@ import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:terrain50str
import array2d_classify_convert_bin from '../../manip/array2d_classify_convert_bin.mjs'; import array2d_classify_convert_bin from '../../manip/array2d_classify_convert_bin.mjs';
class Terrain50StreamReader { class Terrain50StreamReader {
constructor(threshold, tolerant = false) { constructor(threshold = 0.1, tolerant = false) {
this.threshold = threshold; this.threshold = threshold;
this.tolerant = tolerant; this.tolerant = tolerant;

View file

@ -10,5 +10,6 @@ export default function(cli) {
.map(el => parseInt(el)) .map(el => parseInt(el))
.reverse(); .reverse();
}) })
.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer"); .argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer")
.argument("output", "The path to the directory to write the generated TFRecord files to.", null, "string");
} }

View file

@ -1,6 +1,12 @@
"use strict"; "use strict";
import fs from 'fs';
import settings from '../../settings.mjs'; import settings from '../../settings.mjs';
import TFRecordWriter from '../../lib/io/TFRecordWriter.mjs';
import RadarReader from '../../lib/io/RadarReader.mjs';
import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs';
export default async function() { export default async function() {
if(typeof settings.water !== "string") if(typeof settings.water !== "string")
@ -9,6 +15,15 @@ export default async function() {
if(typeof settings.rainfall !== "string") if(typeof settings.rainfall !== "string")
throw new Error(`Error: No filepath to rainfall radar data specified.`); throw new Error(`Error: No filepath to rainfall radar data specified.`);
if(typeof settings.output !== "string")
throw new Error(`Error: No output directory specified.`);
// TODO: Do the fanceh parsing stuff here if(!fs.existsSync(settings.output))
await fs.promises.mkdir(settings.output, { recursive: true });
const writer = new TFRecordWriter(settings.output, settings.count_file);
const reader_radar = new RadarReader();
const reader_water = new Terrain50StreamReader();
await writer.write(reader_radar, reader_water);
} }