sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-06-30 18:44:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

ds_eo: lower memory usage

Browse source
This commit is contained in:
Starbeamrainbowlabs 2023-01-20 18:55:52 +00:00
parent b5e68fc1a3
commit 65a2e16a4c
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
1 changed files with 1 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
aimodel/src/lib/dataset/dataset_encoderonly.py
View file

@ -116,9 +116,7 @@ def make_dataset(filepaths, compression_type="GZIP", parallel_reads_multiplier=3
compression_type=compression_type,
num_parallel_reads=math.ceil(os.cpu_count() * parallel_reads_multiplier) if parallel_reads_multiplier > 0 else None
)
if shuffle:
dataset = dataset.shuffle(128) # additional shuffle buffer to mix things up
# If we want another shuffle buffer here, we'd need to split the map function which we don't really want to do
dataset = dataset.map(parse_item(heightmap=heightmap, **kwargs), num_parallel_calls=tf.data.AUTOTUNE) \
.unbatch()