From af26964c6af038c7c92f799771be70b5a11b3336 Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Fri, 21 Oct 2022 15:35:43 +0100 Subject: [PATCH] batched_iterator: reset i_item after every time --- aimodel/src/lib/dataset/batched_iterator.py | 3 ++- aimodel/src/subcommands/train_predict.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/aimodel/src/lib/dataset/batched_iterator.py b/aimodel/src/lib/dataset/batched_iterator.py index 3c45dc2..0a40f17 100644 --- a/aimodel/src/lib/dataset/batched_iterator.py +++ b/aimodel/src/lib/dataset/batched_iterator.py @@ -16,5 +16,6 @@ def batched_iterator(dataset, tensors_in_item=1, batch_size=64): if i_item >= batch_size: yield [ tf.stack(tensors) for tensors in acc ] + i_item = 0 for arr in acc: - arr.clear() + arr.clear() \ No newline at end of file diff --git a/aimodel/src/subcommands/train_predict.py b/aimodel/src/subcommands/train_predict.py index 3bce9fe..0d2b8ba 100644 --- a/aimodel/src/subcommands/train_predict.py +++ b/aimodel/src/subcommands/train_predict.py @@ -118,7 +118,7 @@ def do_png(args, ai, dataset, model_code): i += 1 if i % 100 == 0: - sys.stderr.write(f"Processed {i} items") + sys.stderr.write(f"Processed {i} items\r") def do_jsonl(args, ai, dataset): write_mode = "wt" if args.output.endswith(".gz") else "w"