Fix many bugs

Many bugs remain though
This commit is contained in:
Starbeamrainbowlabs 2022-05-19 17:54:14 +01:00
parent cc5efbae8a
commit bb018c53f6
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
11 changed files with 2124 additions and 17 deletions

File diff suppressed because it is too large Load diff

View file

@ -13,6 +13,7 @@
"author": "Starbeamrainbowlabs",
"license": "GPL-3.0-or-later",
"dependencies": {
"@tensorflow/tfjs-node-gpu": "^3.17.0",
"applause-cli": "^1.8.1",
"gunzip-maybe": "^1.4.2",
"pretty-ms": "^7.0.1",

View file

@ -22,7 +22,7 @@ async function load_subcommands(cli) {
export default async function () {
let cli = new CliParser(path.resolve(__dirname, "../package.json"));
cli.argument("verbose", "Enable verbose debugging output", null, "boolean")
cli.argument("verbose", "Enable verbose debugging output", false, "boolean")
.argument("log-level", "Sets the log level. Value values: DEBUG, INFO (the default), LOG, WARN, ERROR, NONE", "INFO", "string");
await load_subcommands(cli);
@ -53,7 +53,7 @@ export default async function () {
}
catch(error) {
console.error();
if(settings.cli.verbose)
if(settings.verbose)
throw error;
else
console.error(`${a.fred}${a.hicol}${error.message}${a.reset}`);

View file

@ -1,5 +1,9 @@
"use strict";
import RadarReader from './io/RadarReader.mjs';
import array2d_pool from './manip/array2d_pool.mjs';
class RadarWrangler {
/**
* The total number of timesteps we need in the buffer before we can even consider generating a sample.

View file

@ -6,7 +6,7 @@ import { Readable } from 'stream';
import nexline from 'nexline';
import gunzip from 'gunzip-maybe';
import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:radar");
import log from './NamespacedLog.mjs'; const l = log("reader:radar");
import interpolate from '../manip/array2d_interpolate.mjs';
import transpose from '../manip/array2d_transpose.mjs';

View file

@ -7,14 +7,14 @@ import tfrecord from 'tfrecord-stream';
import pretty_ms from 'pretty-ms';
class TFRecordWriter {
#builder = tfrecord.createBuilder();
constructor(dirpath, count_per_file) {
this.dirpath = dirpath;
this.count_per_file = count_per_file;
if(!fs.existsSync(dirpath))
fs.mkdirSync(dirpath);
this.#builder = tfrecord.createBuilder();
if(!fs.existsSync(this.dirpath))
fs.mkdirSync(this.dirpath);
}
async write(reader_radar, reader_water) {
@ -28,7 +28,7 @@ class TFRecordWriter {
// Start writing to a new file when necessary
if(writer == null || count_this_file > this.count_per_file) {
if(writer !== null) await writer.close();
const filepath_next = path.join(dirpath, `${i_file}.tfrecord`);
const filepath_next = path.join(this.dirpath, `${i_file}.tfrecord`);
writer = await tfrecord.Writer.createFromStream(
fs.createWriteStream(filepath_next)
);

View file

@ -6,9 +6,9 @@ import fs from 'fs';
import path from 'path';
import log from './lib/io/NamespacedLog.mjs'; const l = log("reader:terrain50stream");
import log from './NamespacedLog.mjs'; const l = log("reader:terrain50stream");
import array2d_classify_convert_bin from '../../manip/array2d_classify_convert_bin.mjs';
import array2d_classify_convert_bin from '../manip/array2d_classify_convert_bin.mjs';
class Terrain50StreamReader {
constructor(threshold = 0.1, tolerant = false) {

View file

@ -4,12 +4,12 @@ export default function(cli) {
cli.subcommand("tfrecordify", "Converts rianfall radar and water depth data to a directory of tfrecord files.")
.argument("water", "Path to the water depths file, formatted as a stream of terrain50 objects. May or may not be gzipped.", null, "string")
.argument("rainfall", "Path to the rainfall radar data, formatted as jsonl. May or may not be gzipped.", null, "string")
.argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number.", 64*64)
.argument("rainfall-pattern", "The pattern of the number of time steps to average, as a comma-separated list of numbers. Given a point in time, each successive number specified works BACKWARDS from that point. For example, 1,4,10 would be 3 channels: 1 time step on it's own, then average the next 4 time steps, then average the next 10 steps.", "1,3,3,5,12,24,48", function(value) {
.argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number (default: 4096).", 64*64)
.argument("rainfall-pattern", "The pattern of the number of time steps to average, as a comma-separated list of numbers. Given a point in time, each successive number specified works BACKWARDS from that point. For example, 1,4,10 would be 3 channels: 1 time step on it's own, then average the next 4 time steps, then average the next 10 steps (default: 1,3,3,5,12,24,48).", [1,3,3,5,12,24,48], function(value) {
return value.split(",")
.map(el => parseInt(el))
.reverse();
})
.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data.", 1, "integer")
.argument("water-offset", "Make the water depth data be this many time steps ahead of the rainfall radar data. (default: 1)", 1, "integer")
.argument("output", "The path to the directory to write the generated TFRecord files to.", null, "string");
}

View file

@ -4,7 +4,7 @@ import fs from 'fs';
import settings from '../../settings.mjs';
import TFRecordWriter from '../../lib/io/TFRecordWriter.mjs';
import RadarReader from '../../lib/io/RadarReader.mjs';
import RadarWrangler from '../../lib/RadarWrangler.mjs';
import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs';
@ -20,10 +20,11 @@ export default async function() {
if(!fs.existsSync(settings.output))
await fs.promises.mkdir(settings.output, { recursive: true });
console.log("DEBUG", settings);
const writer = new TFRecordWriter(settings.output, settings.count_file);
const reader_radar = new RadarReader();
const reader_radar = new RadarWrangler(settings.rainfall_pattern);
const reader_water = new Terrain50StreamReader();
await writer.write(reader_radar, reader_water);
await writer.write(reader_radar.iterate(settings.rainfall), reader_water.iterate(settings.water));
}