sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-22 09:13:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

recompress files in the right order

Browse source
This commit is contained in:
Starbeamrainbowlabs 2022-07-25 18:44:23 +01:00
parent 3332fa598a
commit 927c30e189
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
2 changed files with 9 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
rainfallwrangler/src/lib/record/records_recompress.mjs
View file

@ -15,11 +15,17 @@ async function records_recompress(dirpath_source, dirpath_target, items_per_file
.filter(filename => filename.endsWith(`.jsonl.gz`)) .filter(filename => filename.endsWith(`.jsonl.gz`))
.map(filename => path.join(dirpath_source, filename)); .map(filename => path.join(dirpath_source, filename));
files.sort((a, b) => {
let ai = parseInt(a.split(".")[0], 10), bi = parseInt(b.split(".")[0], 10);
if(ai === bi) return 0;
else return ai > bi ? 1 : -1;
});
const reader = nexline({ const reader = nexline({
input: files.map(filepath => new Readable().wrap(fs.createReadStream(filepath).pipe(gunzip()))) input: files.map(filepath => new Readable().wrap(fs.createReadStream(filepath).pipe(gunzip())))
}); });
if(!fs.existsSync(dirpath_target)) if(!fs.existsSync(dirpath_target))
await fs.promises.mkdir(dirpath_target, { recursive: true }); await fs.promises.mkdir(dirpath_target, { recursive: true });

2
rainfallwrangler/src/subcommands/recompress/recompress.mjs
View file

@ -17,5 +17,5 @@ export default async function() {
if(!fs.existsSync(settings.target)) if(!fs.existsSync(settings.target))
await fs.promises.mkdir(settings.target); await fs.promises.mkdir(settings.target);
await records_recompress(settings.source, settings.target); await records_recompress(settings.source, settings.target, settings.count_file);
} }