From c4d3c16873e6fb19594d41062fee8b55710c1878 Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Tue, 27 Sep 2022 18:10:58 +0100 Subject: [PATCH] add some logging --- aimodel/src/subcommands/pretrain_predict.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aimodel/src/subcommands/pretrain_predict.py b/aimodel/src/subcommands/pretrain_predict.py index 14be9c8..d0115e1 100644 --- a/aimodel/src/subcommands/pretrain_predict.py +++ b/aimodel/src/subcommands/pretrain_predict.py @@ -71,6 +71,8 @@ def run(args): output_mode = MODE_TFRECORD if filepath_output.endswith(".tfrecord") or filepath_output.endswith(".tfrecord.gz") else MODE_JSONL + logger.info("Output mode is "+("TFRECORD" if output_mode == MODE_TFRECORD else "JSONL")) + write_mode = "wt" if filepath_output.endswith(".gz") else "w" if output_mode == MODE_TFRECORD: write_mode = "wb" @@ -79,11 +81,13 @@ def run(args): filepath_params = None if filepath_output != "-": handle = handle_open( - filepath_output if args.records_per_file <= 0 else filepath_output.replace("$d", 0), + filepath_output if args.records_per_file <= 0 else filepath_output.replace("$d", str(0)), write_mode ) filepath_params = os.path.join(os.path.dirname(filepath_output), "params.json") + logger.info("filepath_output: {filepath_output}") + logger.info("filepath_params: {filepath_params}") i = 0 i_file = i