diff --git a/aimodel/src/lib/ai/components/CallbackNBatchCsv.py b/aimodel/src/lib/ai/components/CallbackNBatchCsv.py new file mode 100644 index 0000000..de7491b --- /dev/null +++ b/aimodel/src/lib/ai/components/CallbackNBatchCsv.py @@ -0,0 +1,30 @@ +import tensorflow as tf + +from lib.io.handle_open import handle_open + +class CallbackNBatchCsv(tf.keras.callbacks.Callback): + def __init__(self, filepath, n_batches=1, separator="\t", **kwargs) -> None: + super().__init__(**kwargs) + + self.n_batches = n_batches + self.separator = separator + + self.handle = handle_open(filepath) + + + self.batches_seen = 0 + self.keys = None + + def write_header(self, logs): # logs = metrics + self.keys = logs.keys() + self.keys.sort() + self.handle.write("\t".join(self.keys)+"\n") + + def on_batch_end(self, batch, logs=None): # logs = metrics + if self.batches_seen == 0: + self.write_header(logs) + + if self.batches_seen % self.n_batches == 0: + self.handle.write(self.separator.join([str(logs[key]) for key in self.keys]) + "\n") + + self.batches_seen += 1 diff --git a/aimodel/src/lib/ai/helpers/make_callbacks.py b/aimodel/src/lib/ai/helpers/make_callbacks.py index f5fadd2..8d3f5d8 100644 --- a/aimodel/src/lib/ai/helpers/make_callbacks.py +++ b/aimodel/src/lib/ai/helpers/make_callbacks.py @@ -3,10 +3,12 @@ import os import tensorflow as tf from ..components.CallbackCustomModelCheckpoint import CallbackCustomModelCheckpoint +from ..components.CallbackNBatchCsv import CallbackNBatchCsv def make_callbacks(dirpath, model_predict): dirpath_checkpoints = os.path.join(dirpath, "checkpoints") filepath_metrics = os.path.join(dirpath, "metrics.tsv") + filepath_metrics_batch = os.path.join(dirpath, "metrics_batch64.tsv") if not os.path.exists(dirpath_checkpoints): os.mkdir(dirpath_checkpoints) @@ -24,5 +26,9 @@ def make_callbacks(dirpath, model_predict): filename=filepath_metrics, separator="\t" ), + CallbackNBatchCsv( + filepath=filepath_metrics_batch, + n_batches=64 + ), tf.keras.callbacks.ProgbarLogger(count_mode="steps") # batches ] \ No newline at end of file