sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-21 17:03:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

Implement tfrecodify subcommand.

Browse source 
It's all still untested, but that's the next step
This commit is contained in:
Starbeamrainbowlabs 2022-05-19 17:15:15 +01:00
parent 0fa7ae9d6a
commit cc5efbae8a
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
6 changed files with 61 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
rainfallwrangler/package-lock.json generated
View file

@ -11,6 +11,7 @@
"dependencies": {
"applause-cli": "^1.8.1",
"gunzip-maybe": "^1.4.2",
"pretty-ms": "^7.0.1",
"terrain50": "^1.10.1",
"tfrecord-stream": "^0.2.0"
}
@ -274,6 +275,14 @@
"resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
"integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU="
},
"node_modules/parse-ms": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz",
"integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==",
"engines": {
"node": ">=6"
}
},
"node_modules/peek-stream": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz",
@ -284,6 +293,20 @@
"through2": "^2.0.3"
}
},
"node_modules/pretty-ms": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz",
"integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==",
"dependencies": {
"parse-ms": "^2.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@ -698,6 +721,11 @@
"resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
"integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU="
},
"parse-ms": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz",
"integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA=="
},
"peek-stream": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz",
@ -708,6 +736,14 @@
"through2": "^2.0.3"
}
},
"pretty-ms": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz",
"integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==",
"requires": {
"parse-ms": "^2.1.0"
}
},
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",

1
rainfallwrangler/package.json
View file

@ -15,6 +15,7 @@
"dependencies": {
"applause-cli": "^1.8.1",
"gunzip-maybe": "^1.4.2",
"pretty-ms": "^7.0.1",
"terrain50": "^1.10.1",
"tfrecord-stream": "^0.2.0"
}

7
rainfallwrangler/src/lib/io/TFRecordWriter.mjs
View file

@ -4,6 +4,7 @@ import fs from 'fs';
import path from 'path';
import tfrecord from 'tfrecord-stream';
import pretty_ms from 'pretty-ms';
class TFRecordWriter {
constructor(dirpath, count_per_file) {
@ -16,11 +17,11 @@ class TFRecordWriter {
this.#builder = tfrecord.createBuilder();
}
write(reader_radar, reader_water) {
async write(reader_radar, reader_water) {
// TODO: Shuffle stuff about in the *Python* data pipeline
let writer = null;
let i = -1, i_file = 0, count_this_file = 0;
let i = -1, i_file = 0, count_this_file = 0, time_start = new Date();
while(true) {
i++;
@ -45,6 +46,8 @@ class TFRecordWriter {
);
await writer.writeExample(example_next);
process.stderr.write(`Elapsed: ${pretty_ms(new Date() - time_start)}, Written ${count_this_file}/${i_file}/${i} examples/files/total\r`);
}
}

2
rainfallwrangler/src/lib/io/Terrain50StreamReader.mjs
View file

@ -11,7 +11,7 @@ import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:terrain50str
import array2d_classify_convert_bin from '../../manip/array2d_classify_convert_bin.mjs';
class Terrain50StreamReader {
constructor(threshold, tolerant = false) {
constructor(threshold = 0.1, tolerant = false) {
this.threshold = threshold;
this.tolerant = tolerant;

3
rainfallwrangler/src/subcommands/tfrecordify/meta.mjs
View file

@ -10,5 +10,6 @@ export default function(cli) {
.map(el => parseInt(el))
.reverse();
})
.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer");
.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer")
.argument("output", "The path to the directory to write the generated TFRecord files to.", null, "string");
}

17
rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs
View file

@ -1,6 +1,12 @@
"use strict";
import fs from 'fs';
import settings from '../../settings.mjs';
import TFRecordWriter from '../../lib/io/TFRecordWriter.mjs';
import RadarReader from '../../lib/io/RadarReader.mjs';
import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs';
export default async function() {
if(typeof settings.water !== "string")
@ -9,6 +15,15 @@ export default async function() {
if(typeof settings.rainfall !== "string")
throw new Error(`Error: No filepath to rainfall radar data specified.`);
if(typeof settings.output !== "string")
throw new Error(`Error: No output directory specified.`);
// TODO: Do the fanceh parsing stuff here
if(!fs.existsSync(settings.output))
await fs.promises.mkdir(settings.output, { recursive: true });
const writer = new TFRecordWriter(settings.output, settings.count_file);
const reader_radar = new RadarReader();
const reader_water = new Terrain50StreamReader();
await writer.write(reader_radar, reader_water);
}