sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-25 10:32:59 +00:00
Code Issues Projects Releases Packages Wiki Activity

dataset: drop incomplete batches

Browse source
This commit is contained in:
Starbeamrainbowlabs 2022-09-05 15:36:10 +01:00
parent c94f5d042e
commit 9d39215dd5
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
aimodel/src/lib/dataset/dataset.py
View file

@ -51,7 +51,7 @@ def make_dataset(filenames, metadata, shape_watch_desired=[100,100], compression
num_parallel_reads=math.ceil(os.cpu_count() * parallel_reads_multiplier) num_parallel_reads=math.ceil(os.cpu_count() * parallel_reads_multiplier)
).shuffle(shuffle_buffer_size) \ ).shuffle(shuffle_buffer_size) \
.map(parse_item(metadata, shape_water_desired=shape_watch_desired), num_parallel_calls=tf.data.AUTOTUNE) \ .map(parse_item(metadata, shape_water_desired=shape_watch_desired), num_parallel_calls=tf.data.AUTOTUNE) \
.batch(batch_size) \ .batch(batch_size, drop_remainder=True) \
.prefetch(0 if "NO_PREFETCH" in os.environ else tf.data.AUTOTUNE) .prefetch(0 if "NO_PREFETCH" in os.environ else tf.data.AUTOTUNE)