sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-22 09:13:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

.tfrecord files are too much hassle

Browse source 
let's go with a standard of .jsonl.gz instead
This commit is contained in:
Starbeamrainbowlabs 2022-07-01 18:28:39 +01:00
parent f5f267c6b6
commit 1297f41105
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
10 changed files with 161 additions and 290 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
.gitignore vendored
View file

@ -339,3 +339,32 @@ cython_debug/
*.code-workspace *.code-workspace
# End of https://www.toptal.com/developers/gitignore/api/python,node,git,visualstudiocode # End of https://www.toptal.com/developers/gitignore/api/python,node,git,visualstudiocode
# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode
# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode
### VisualStudioCode ###
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/*.code-snippets
# Local History for Visual Studio Code
.history/
# Built Visual Studio Code Extensions
*.vsix
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
.ionide
# Support for Project snippet scope
.vscode/*.code-snippets
# Ignore code-workspaces
*.code-workspace
# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode

2
.vscode/settings.json vendored Normal file
View file

@ -0,0 +1,2 @@
{
}

279
rainfallwrangler/package-lock.json generated
View file

@ -13,8 +13,8 @@
"applause-cli": "^1.8.1", "applause-cli": "^1.8.1",
"gunzip-maybe": "^1.4.2", "gunzip-maybe": "^1.4.2",
"pretty-ms": "^8.0.0", "pretty-ms": "^8.0.0",
"terrain50": "^1.10.1", "spawn-stream": "^1.0.2",
"tfrecord-stream": "^0.2.0" "terrain50": "^1.10.1"
} }
}, },
"node_modules/@mapbox/node-pre-gyp": { "node_modules/@mapbox/node-pre-gyp": {
@ -163,60 +163,6 @@
"node": ">= 10" "node": ">= 10"
} }
}, },
"node_modules/@protobufjs/aspromise": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
"integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78="
},
"node_modules/@protobufjs/base64": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
},
"node_modules/@protobufjs/codegen": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
"integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
},
"node_modules/@protobufjs/eventemitter": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
"integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A="
},
"node_modules/@protobufjs/fetch": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
"integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=",
"dependencies": {
"@protobufjs/aspromise": "^1.1.1",
"@protobufjs/inquire": "^1.1.0"
}
},
"node_modules/@protobufjs/float": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
"integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E="
},
"node_modules/@protobufjs/inquire": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
"integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik="
},
"node_modules/@protobufjs/path": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
"integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0="
},
"node_modules/@protobufjs/pool": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
"integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q="
},
"node_modules/@protobufjs/utf8": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
"integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA="
},
"node_modules/@tensorflow/tfjs": { "node_modules/@tensorflow/tfjs": {
"version": "3.18.0", "version": "3.18.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-3.18.0.tgz", "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-3.18.0.tgz",
@ -475,25 +421,11 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
}, },
"node_modules/awaitify-stream": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/awaitify-stream/-/awaitify-stream-1.0.2.tgz",
"integrity": "sha512-JE6mrRIPxhBQWt9Mu4u2XJF9V9xp4p5+Uxif/Ol/s2TFC/9+offUE50j/KQdaVWwMyGlKrlsE7Ncq/yhc6AJ+w=="
},
"node_modules/balanced-match": { "node_modules/balanced-match": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
}, },
"node_modules/bindings": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
"optional": true,
"dependencies": {
"file-uri-to-path": "1.0.0"
}
},
"node_modules/brace-expansion": { "node_modules/brace-expansion": {
"version": "1.1.11", "version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -723,20 +655,6 @@
"node": ">=6" "node": ">=6"
} }
}, },
"node_modules/fast-crc32c": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-crc32c/-/fast-crc32c-2.0.0.tgz",
"integrity": "sha512-LIREwygxtxzHF11oLJ4xIVKu/ZWNgrj/QaGvaSD8ZggIsgCyCtSYevlrpWVqNau57ZwezV8K1HFBSjQ7FcRbTQ==",
"optionalDependencies": {
"sse4_crc32": "^6.0.1"
}
},
"node_modules/file-uri-to-path": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
"optional": true
},
"node_modules/form-data": { "node_modules/form-data": {
"version": "3.0.1", "version": "3.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
@ -1067,12 +985,6 @@
"proj4": "^2.3.10" "proj4": "^2.3.10"
} }
}, },
"node_modules/node-addon-api": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-1.7.2.tgz",
"integrity": "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==",
"optional": true
},
"node_modules/node-fetch": { "node_modules/node-fetch": {
"version": "2.6.7", "version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
@ -1211,31 +1123,6 @@
"wkt-parser": "^1.3.1" "wkt-parser": "^1.3.1"
} }
}, },
"node_modules/protobufjs": {
"version": "6.11.3",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz",
"integrity": "sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==",
"hasInstallScript": true,
"dependencies": {
"@protobufjs/aspromise": "^1.1.2",
"@protobufjs/base64": "^1.1.2",
"@protobufjs/codegen": "^2.0.4",
"@protobufjs/eventemitter": "^1.1.0",
"@protobufjs/fetch": "^1.1.0",
"@protobufjs/float": "^1.0.2",
"@protobufjs/inquire": "^1.1.0",
"@protobufjs/path": "^1.1.2",
"@protobufjs/pool": "^1.1.0",
"@protobufjs/utf8": "^1.1.0",
"@types/long": "^4.0.1",
"@types/node": ">=13.7.0",
"long": "^4.0.0"
},
"bin": {
"pbjs": "bin/pbjs",
"pbts": "bin/pbts"
}
},
"node_modules/pump": { "node_modules/pump": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
@ -1333,25 +1220,16 @@
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="
}, },
"node_modules/spawn-stream": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/spawn-stream/-/spawn-stream-1.0.2.tgz",
"integrity": "sha512-gRlPPUISTDoeWqQSJNnqAp9vjHc7b7m3aWRgp1dzL3c183f5Xx2lnVtvPRCQy+pyqfDSx5DZa95Yh7XtPuJE0Q=="
},
"node_modules/sprintf-js": { "node_modules/sprintf-js": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="
}, },
"node_modules/sse4_crc32": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/sse4_crc32/-/sse4_crc32-6.0.1.tgz",
"integrity": "sha512-FUTYXpLroqytNKWIfHzlDWoy9E4tmBB/RklNMy6w3VJs+/XEYAHgbiylg4SS43iOk/9bM0BlJ2EDpFAGT66IoQ==",
"hasInstallScript": true,
"optional": true,
"dependencies": {
"bindings": "^1.3.0",
"node-addon-api": "^1.3.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/stream-shift": { "node_modules/stream-shift": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
@ -1450,16 +1328,6 @@
"nnng": "^1.0.0" "nnng": "^1.0.0"
} }
}, },
"node_modules/tfrecord-stream": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/tfrecord-stream/-/tfrecord-stream-0.2.0.tgz",
"integrity": "sha512-B7TtEQUqR6u/0drbPZBUUaBFVp4QbEPBgSiUolSarKuyIYqM6AAgB/Tre5gECoSacEl39hgHBnXm+xjj9zzIcA==",
"dependencies": {
"awaitify-stream": "^1.0.2",
"fast-crc32c": "^2.0.0",
"protobufjs": "^6.8.4"
}
},
"node_modules/through2": { "node_modules/through2": {
"version": "2.0.5", "version": "2.0.5",
"resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz",
@ -1767,60 +1635,6 @@
} }
} }
}, },
"@protobufjs/aspromise": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
"integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78="
},
"@protobufjs/base64": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
},
"@protobufjs/codegen": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
"integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
},
"@protobufjs/eventemitter": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
"integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A="
},
"@protobufjs/fetch": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
"integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=",
"requires": {
"@protobufjs/aspromise": "^1.1.1",
"@protobufjs/inquire": "^1.1.0"
}
},
"@protobufjs/float": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
"integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E="
},
"@protobufjs/inquire": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
"integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik="
},
"@protobufjs/path": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
"integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0="
},
"@protobufjs/pool": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
"integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q="
},
"@protobufjs/utf8": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
"integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA="
},
"@tensorflow/tfjs": { "@tensorflow/tfjs": {
"version": "3.18.0", "version": "3.18.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-3.18.0.tgz", "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-3.18.0.tgz",
@ -2040,25 +1854,11 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
}, },
"awaitify-stream": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/awaitify-stream/-/awaitify-stream-1.0.2.tgz",
"integrity": "sha512-JE6mrRIPxhBQWt9Mu4u2XJF9V9xp4p5+Uxif/Ol/s2TFC/9+offUE50j/KQdaVWwMyGlKrlsE7Ncq/yhc6AJ+w=="
},
"balanced-match": { "balanced-match": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
}, },
"bindings": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
"optional": true,
"requires": {
"file-uri-to-path": "1.0.0"
}
},
"brace-expansion": { "brace-expansion": {
"version": "1.1.11", "version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -2246,20 +2046,6 @@
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
}, },
"fast-crc32c": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-crc32c/-/fast-crc32c-2.0.0.tgz",
"integrity": "sha512-LIREwygxtxzHF11oLJ4xIVKu/ZWNgrj/QaGvaSD8ZggIsgCyCtSYevlrpWVqNau57ZwezV8K1HFBSjQ7FcRbTQ==",
"requires": {
"sse4_crc32": "^6.0.1"
}
},
"file-uri-to-path": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
"optional": true
},
"form-data": { "form-data": {
"version": "3.0.1", "version": "3.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
@ -2534,12 +2320,6 @@
"proj4": "^2.3.10" "proj4": "^2.3.10"
} }
}, },
"node-addon-api": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-1.7.2.tgz",
"integrity": "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==",
"optional": true
},
"node-fetch": { "node-fetch": {
"version": "2.6.7", "version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
@ -2637,26 +2417,6 @@
"wkt-parser": "^1.3.1" "wkt-parser": "^1.3.1"
} }
}, },
"protobufjs": {
"version": "6.11.3",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz",
"integrity": "sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==",
"requires": {
"@protobufjs/aspromise": "^1.1.2",
"@protobufjs/base64": "^1.1.2",
"@protobufjs/codegen": "^2.0.4",
"@protobufjs/eventemitter": "^1.1.0",
"@protobufjs/fetch": "^1.1.0",
"@protobufjs/float": "^1.0.2",
"@protobufjs/inquire": "^1.1.0",
"@protobufjs/path": "^1.1.2",
"@protobufjs/pool": "^1.1.0",
"@protobufjs/utf8": "^1.1.0",
"@types/long": "^4.0.1",
"@types/node": ">=13.7.0",
"long": "^4.0.0"
}
},
"pump": { "pump": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
@ -2742,21 +2502,16 @@
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="
}, },
"spawn-stream": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/spawn-stream/-/spawn-stream-1.0.2.tgz",
"integrity": "sha512-gRlPPUISTDoeWqQSJNnqAp9vjHc7b7m3aWRgp1dzL3c183f5Xx2lnVtvPRCQy+pyqfDSx5DZa95Yh7XtPuJE0Q=="
},
"sprintf-js": { "sprintf-js": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="
}, },
"sse4_crc32": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/sse4_crc32/-/sse4_crc32-6.0.1.tgz",
"integrity": "sha512-FUTYXpLroqytNKWIfHzlDWoy9E4tmBB/RklNMy6w3VJs+/XEYAHgbiylg4SS43iOk/9bM0BlJ2EDpFAGT66IoQ==",
"optional": true,
"requires": {
"bindings": "^1.3.0",
"node-addon-api": "^1.3.0"
}
},
"stream-shift": { "stream-shift": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
@ -2831,16 +2586,6 @@
"nnng": "^1.0.0" "nnng": "^1.0.0"
} }
}, },
"tfrecord-stream": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/tfrecord-stream/-/tfrecord-stream-0.2.0.tgz",
"integrity": "sha512-B7TtEQUqR6u/0drbPZBUUaBFVp4QbEPBgSiUolSarKuyIYqM6AAgB/Tre5gECoSacEl39hgHBnXm+xjj9zzIcA==",
"requires": {
"awaitify-stream": "^1.0.2",
"fast-crc32c": "^2.0.0",
"protobufjs": "^6.8.4"
}
},
"through2": { "through2": {
"version": "2.0.5", "version": "2.0.5",
"resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz",

4
rainfallwrangler/package.json
View file

@ -17,7 +17,7 @@
"applause-cli": "^1.8.1", "applause-cli": "^1.8.1",
"gunzip-maybe": "^1.4.2", "gunzip-maybe": "^1.4.2",
"pretty-ms": "^8.0.0", "pretty-ms": "^8.0.0",
"terrain50": "^1.10.1", "spawn-stream": "^1.0.2",
"tfrecord-stream": "^0.2.0" "terrain50": "^1.10.1"
} }
} }

32
rainfallwrangler/src/lib/io/TFRecordWriter.mjs → rainfallwrangler/src/lib/io/RecordWrangler.mjs
View file

@ -3,11 +3,12 @@
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import tfrecord from 'tfrecord-stream'; import RecordBuilder from '../record/RecordBuilder.mjs';
import RecordsWriter from '../record/RecordsWriter.mjs';
import pretty_ms from 'pretty-ms'; import pretty_ms from 'pretty-ms';
class TFRecordWriter { class RecordWrangler {
#builder = tfrecord.createBuilder(); #builder = new RecordBuilder();
constructor(dirpath, count_per_file) { constructor(dirpath, count_per_file) {
this.dirpath = dirpath; this.dirpath = dirpath;
@ -25,13 +26,14 @@ class TFRecordWriter {
while(true) { while(true) {
i++; i++;
console.log(`RecordWriter step ${i}`);
// Start writing to a new file when necessary // Start writing to a new file when necessary
if(writer == null || count_this_file > this.count_per_file) { if(writer == null || count_this_file > this.count_per_file) {
if(writer !== null) await writer.close(); if(writer !== null) await writer.close();
const filepath_next = path.join(this.dirpath, `${i_file}.tfrecord`); const filepath_next = path.join(this.dirpath, `${i_file}.jsonl.gz`);
writer = await tfrecord.Writer.createFromStream( writer = new RecordsWriter(filepath_next);
fs.createWriteStream(filepath_next) console.log(`RecordWriter NEW FILE ${filepath_next}`);
);
i_file++; i_file++;
} }
@ -45,7 +47,7 @@ class TFRecordWriter {
sample_water.value sample_water.value
); );
await writer.writeExample(example_next); await writer.write(example_next);
process.stderr.write(`Elapsed: ${pretty_ms(new Date() - time_start)}, Written ${count_this_file}/${i_file}/${i} examples/files/total\r`); process.stderr.write(`Elapsed: ${pretty_ms(new Date() - time_start)}, Written ${count_this_file}/${i_file}/${i} examples/files/total\r`);
} }
@ -53,16 +55,10 @@ class TFRecordWriter {
} }
make_example(sample_radar, sample_water) { make_example(sample_radar, sample_water) {
console.log(`SAMPLE WATER ${sample_water.flat().length} RAINFALL ${sample_radar.flat().length}`); this.#builder.add("rainfallradar", sample_radar);
const sample_radar_flat1 = sample_radar.flat(); this.#builder.add("waterdepth", sample_water.flat);
this.#builder.setFloats("rainfallradar", sample_radar_flat1.flat()); return this.#builder.release();
this.#builder.setInteger("rainfallradar_width", sample_radar[0].length);
this.#builder.setInteger("rainfallradar_channelsize", sample_radar_flat1[0].length);
this.#builder.setFloats("waterdepth", sample_water.flat());
this.#builder.setInteger("waterdepth_width", sample_water[0].length);
return this.#builder.releaseExample();
} }
} }
export default TFRecordWriter; export default RecordWrangler;

52
rainfallwrangler/src/lib/io/StreamHelpers.mjs Normal file
View file

@ -0,0 +1,52 @@
"use strict";
/**
* Writes data to a stream, automatically waiting for the drain event if asked.
* See also write_safe.
* @param {stream.Writable} stream_out The writable stream to write to.
* @param {string|Buffer|Uint8Array} data The data to write.
* @return {Promise} A promise that resolves when writing is complete.
* @private
*/
function write_safe(stream_out, data) {
return new Promise(function (resolve, reject) {
// Handle errors
let handler_error = (error) => {
stream_out.off("error", handler_error);
reject(error);
};
stream_out.on("error", handler_error);
if(typeof data == "string" ? stream_out.write(data, "utf-8") : stream_out.write(data)) {
// We're good to go
stream_out.off("error", handler_error);
resolve();
}
else {
// We need to wait for the drain event before continuing
stream_out.once("drain", () => {
stream_out.off("error", handler_error);
resolve();
});
}
});
}
/**
* Waits for the given stream to end and finish writing data.
* NOTE: This function is not tested and guaranteed yet. (ref #10 the HydroIndexWriter bug)
* @param {stream.Writable} stream The stream to end.
* @param {Buffer|string} [chunk=undefined] Optional. A chunk to write when calling .end().
* @return {Promise} A Promise that resolves when writing is complete.
* @private
*/
function end_safe(stream, chunk = undefined) {
return new Promise((resolve, _reject) => {
stream.once("finish", resolve);
if(typeof chunk == "undefined") stream.end();
else stream.end(chunk);
});
}
export { write_safe, end_safe };

19
rainfallwrangler/src/lib/record/RecordBuilder.mjs Normal file
View file

@ -0,0 +1,19 @@
"use strict";
class RecordBuilder {
constructor() {
this.acc = new Map();
}
add(key, value) {
this.acc.set(key, value);
}
release() {
const result = this.acc;
this.acc = new Map();
return result;
}
}
export default RecordBuilder;

28
rainfallwrangler/src/lib/record/RecordsWriter.mjs Normal file
View file

@ -0,0 +1,28 @@
"use strict";
import fs from 'fs';
import SpawnStream from 'spawn-stream';
import { write_safe, end_safe } from '../io/StreamHelpers.mjs';
class RecordsWriter {
#stream_out = fs.createWriteStream(filepath);
#gzip = SpawnStream("gzip");
constructor(filepath) {
this.#gzip.pipe(this.#stream_out);
}
async write(sample) {
console.log(sample);
await write_safe(this.#gzip, JSON.stringify(sample));
}
async close() {
await this.#gzip.close();
await this.#stream_out.close();
}
}
export default RecordsWriter;

2
rainfallwrangler/src/subcommands/tfrecordify/meta.mjs → rainfallwrangler/src/subcommands/recordify/meta.mjs
View file

@ -1,7 +1,7 @@
"use strict"; "use strict";
export default function(cli) { export default function(cli) {
cli.subcommand("tfrecordify", "Converts rianfall radar and water depth data to a directory of tfrecord files.") cli.subcommand("recordify", "Converts rainfall radar and water depth data to a directory of .jsonl.gz files.")
.argument("water", "Path to the water depths file, formatted as a stream of terrain50 objects. May or may not be gzipped.", null, "string") .argument("water", "Path to the water depths file, formatted as a stream of terrain50 objects. May or may not be gzipped.", null, "string")
.argument("rainfall", "Path to the rainfall radar data, formatted as jsonl. May or may not be gzipped.", null, "string") .argument("rainfall", "Path to the rainfall radar data, formatted as jsonl. May or may not be gzipped.", null, "string")
.argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number (default: 4096).", 64*64) .argument("count-file", "The number of records to store in each TFRecord file. See the documentation for the optimal value of this number (default: 4096).", 64*64)

4
rainfallwrangler/src/subcommands/tfrecordify/tfrecordify.mjs → rainfallwrangler/src/subcommands/recordify/recordify.mjs
View file

@ -3,7 +3,7 @@
import fs from 'fs'; import fs from 'fs';
import settings from '../../settings.mjs'; import settings from '../../settings.mjs';
import TFRecordWriter from '../../lib/io/TFRecordWriter.mjs'; import RecordWrangler from '../../lib/io/RecordWrangler.mjs';
import RadarWrangler from '../../lib/RadarWrangler.mjs'; import RadarWrangler from '../../lib/RadarWrangler.mjs';
import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs'; import Terrain50StreamReader from '../../lib/io/Terrain50StreamReader.mjs';
@ -22,7 +22,7 @@ export default async function() {
await fs.promises.mkdir(settings.output, { recursive: true }); await fs.promises.mkdir(settings.output, { recursive: true });
console.log("DEBUG", settings); console.log("DEBUG", settings);
const writer = new TFRecordWriter(settings.output, settings.count_file); const writer = new RecordWrangler(settings.output, settings.count_file);
const reader_radar = new RadarWrangler(settings.rainfall_pattern); const reader_radar = new RadarWrangler(settings.rainfall_pattern);
const reader_water = new Terrain50StreamReader(); const reader_water = new Terrain50StreamReader();