mirror of
https://github.com/sbrl/research-rainfallradar
synced 2024-11-21 17:03:00 +00:00
Fix many bugs
Many bugs remain though
This commit is contained in:
parent
cc5efbae8a
commit
bb018c53f6
11 changed files with 2124 additions and 17 deletions
2101
rainfallwrangler/package-lock.json
generated
2101
rainfallwrangler/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
@ -13,6 +13,7 @@
|
|||
"author": "Starbeamrainbowlabs",
|
||||
"license": "GPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@tensorflow/tfjs-node-gpu": "^3.17.0",
|
||||
"applause-cli": "^1.8.1",
|
||||
"gunzip-maybe": "^1.4.2",
|
||||
"pretty-ms": "^7.0.1",
|
||||
|
|
|
@ -22,7 +22,7 @@ async function load_subcommands(cli) {
|
|||
|
||||
export default async function () {
|
||||
let cli = new CliParser(path.resolve(__dirname, "../package.json"));
|
||||
cli.argument("verbose", "Enable verbose debugging output", null, "boolean")
|
||||
cli.argument("verbose", "Enable verbose debugging output", false, "boolean")
|
||||
.argument("log-level", "Sets the log level. Value values: DEBUG, INFO (the default), LOG, WARN, ERROR, NONE", "INFO", "string");
|
||||
|
||||
await load_subcommands(cli);
|
||||
|
@ -53,7 +53,7 @@ export default async function () {
|
|||
}
|
||||
catch(error) {
|
||||
console.error();
|
||||
if(settings.cli.verbose)
|
||||
if(settings.verbose)
|
||||
throw error;
|
||||
else
|
||||
console.error(`${a.fred}${a.hicol}${error.message}${a.reset}`);
|
||||
|
|
|
@ -1,5 +1,9 @@
|
|||
"use strict";
|
||||
|
||||
import RadarReader from './io/RadarReader.mjs';
|
||||
|
||||
import array2d_pool from './manip/array2d_pool.mjs';
|
||||
|
||||
class RadarWrangler {
|
||||
/**
|
||||
* The total number of timesteps we need in the buffer before we can even consider generating a sample.
|
||||
|
|
|
@ -6,7 +6,7 @@ import { Readable } from 'stream';
|
|||
import nexline from 'nexline';
|
||||
import gunzip from 'gunzip-maybe';
|
||||
|
||||
import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:radar");
|
||||
import log from './NamespacedLog.mjs'; const l = log("reader:radar");
|
||||
import interpolate from '../manip/array2d_interpolate.mjs';
|
||||
import transpose from '../manip/array2d_transpose.mjs';
|
||||
|
||||
|
|
|
@ -7,14 +7,14 @@ import tfrecord from 'tfrecord-stream';
|
|||
import pretty_ms from 'pretty-ms';
|
||||
|
||||
class TFRecordWriter {
|
||||
#builder = tfrecord.createBuilder();
|
||||
|
||||
constructor(dirpath, count_per_file) {
|
||||
this.dirpath = dirpath;
|
||||
this.count_per_file = count_per_file;
|
||||
|
||||
if(!fs.existsSync(dirpath))
|
||||
fs.mkdirSync(dirpath);
|
||||
|
||||
this.#builder = tfrecord.createBuilder();
|
||||
if(!fs.existsSync(this.dirpath))
|
||||
fs.mkdirSync(this.dirpath);
|
||||
}
|
||||
|
||||
async write(reader_radar, reader_water) {
|
||||
|
@ -28,7 +28,7 @@ class TFRecordWriter {
|
|||
// Start writing to a new file when necessary
|
||||
if(writer == null || count_this_file > this.count_per_file) {
|
||||
if(writer !== null) await writer.close();
|
||||
const filepath_next = path.join(dirpath, `${i_file}.tfrecord`);
|
||||
const filepath_next = path.join(this.dirpath, `${i_file}.tfrecord`);
|
||||
writer = await tfrecord.Writer.createFromStream(
|
||||
fs.createWriteStream(filepath_next)
|
||||
);
|
||||
|
|
|
@ -6,9 +6,9 @@ import fs from 'fs';
|
|||
import path from 'path';
|
||||
|
||||
|
||||
import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:terrain50stream");
|
||||
import log from './NamespacedLog.mjs'; const l = log("reader:terrain50stream");
|
||||
|
||||
import array2d_classify_convert_bin from '../../manip/array2d_classify_convert_bin.mjs';
|
||||
import array2d_classify_convert_bin from '../manip/array2d_classify_convert_bin.mjs';
|
||||
|
||||
class Terrain50StreamReader {
|
||||
constructor(threshold = 0.1, tolerant = false) {
|
||||
|
|
|
@ -4,12 +4,12 @@ export default function(cli) {
|
|||
cli.subcommand("tfrecordify", "Converts rianfall radar and water depth data to a directory of tfrecord files.")
|
||||
.argument("water", "Path to the water depths file, formatted as a stream of terrain50 objects. May or may not be gzipped.", null, "string")
|
||||
.argument("rainfall", "Path to the rainfall radar data, formatted as jsonl. May or may not be gzipped.", null, "string")
|
||||
.argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number.", 64*64)
|
||||
.argument("rainfall-pattern", "The pattern of the number of time steps to average, as a comma-separated list of numbers. Given a point in time, each successive number specified works BACKWARDS from that point. For example, 1,4,10 would be 3 channels: 1 time step on it's own, then average the next 4 time steps, then average the next 10 steps.", "1,3,3,5,12,24,48", function(value) {
|
||||
.argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number (default: 4096).", 64*64)
|
||||
.argument("rainfall-pattern", "The pattern of the number of time steps to average, as a comma-separated list of numbers. Given a point in time, each successive number specified works BACKWARDS from that point. For example, 1,4,10 would be 3 channels: 1 time step on it's own, then average the next 4 time steps, then average the next 10 steps (default: 1,3,3,5,12,24,48).", [1,3,3,5,12,24,48], function(value) {
|
||||
return value.split(",")
|
||||
.map(el => parseInt(el))
|
||||
.reverse();
|
||||
})
|
||||
.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer")
|
||||
.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data. (default: 1)", 1, "integer")
|
||||
.argument("output", "The path to the directory to write the generated TFRecord files to.", null, "string");
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ import fs from 'fs';
|
|||
|
||||
import settings from '../../settings.mjs';
|
||||
import TFRecordWriter from '../../lib/io/TFRecordWriter.mjs';
|
||||
import RadarReader from '../../lib/io/RadarReader.mjs';
|
||||
import RadarWrangler from '../../lib/RadarWrangler.mjs';
|
||||
import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs';
|
||||
|
||||
|
||||
|
@ -20,10 +20,11 @@ export default async function() {
|
|||
|
||||
if(!fs.existsSync(settings.output))
|
||||
await fs.promises.mkdir(settings.output, { recursive: true });
|
||||
|
||||
|
||||
console.log("DEBUG", settings);
|
||||
const writer = new TFRecordWriter(settings.output, settings.count_file);
|
||||
const reader_radar = new RadarReader();
|
||||
const reader_radar = new RadarWrangler(settings.rainfall_pattern);
|
||||
const reader_water = new Terrain50StreamReader();
|
||||
|
||||
await writer.write(reader_radar, reader_water);
|
||||
await writer.write(reader_radar.iterate(settings.rainfall), reader_water.iterate(settings.water));
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue