sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-22 09:13:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

rainfallwrangler does NOT mess up the ordering of the data

Browse source
This commit is contained in:
Starbeamrainbowlabs 2022-10-18 19:07:14 +01:00
parent f121bfb981
commit 0c11ddca4b
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
4 changed files with 7 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -49,3 +49,5 @@ NOW
``` ```
Note to self: 150.12 hashes/sec on i7-4770 4c8t, ???.?? hashes/sec on Viper compute Note to self: 150.12 hashes/sec on i7-4770 4c8t, ???.?? hashes/sec on Viper compute
After double checking, rainfallwrangler does NOT mess with the ordering of the data.

1
rainfallwrangler/src/subcommands/jsonl2tfrecord/jsonl2tfrecord.mjs
View file

@ -17,5 +17,6 @@ export default async function() {
if(!fs.existsSync(settings.target)) if(!fs.existsSync(settings.target))
await fs.promises.mkdir(settings.target); await fs.promises.mkdir(settings.target);
// Reordering CLEAN, does NOT shuffle
await jsonl_to_tf(settings.source, settings.target); await jsonl_to_tf(settings.source, settings.target);
} }

1
rainfallwrangler/src/subcommands/recompress/recompress.mjs
View file

@ -17,5 +17,6 @@ export default async function() {
if(!fs.existsSync(settings.target)) if(!fs.existsSync(settings.target))
await fs.promises.mkdir(settings.target); await fs.promises.mkdir(settings.target);
// Recompressing CLEAN, does NOT shuffle
await records_recompress(settings.source, settings.target, settings.count_file); await records_recompress(settings.source, settings.target, settings.count_file);
} }

2
rainfallwrangler/src/subcommands/recordify/recordify.mjs
View file

@ -23,6 +23,8 @@ export default async function() {
await fs.promises.mkdir(settings.output, { recursive: true }); await fs.promises.mkdir(settings.output, { recursive: true });
console.log("DEBUG", settings); console.log("DEBUG", settings);
// Recordify CLEAN, does NOT shuffle
const writer = new RecordWrangler(settings.output, settings.count_file); const writer = new RecordWrangler(settings.output, settings.count_file);
const reader_radar = new RadarWrangler(settings.rainfall_pattern); const reader_radar = new RadarWrangler(settings.rainfall_pattern);
const reader_water = new Terrain50StreamReader(); const reader_water = new Terrain50StreamReader();