research-rainfallradar/aimodel/slurm-pretrain-predict.job

92 lines
3 KiB
Text
Raw Normal View History

2022-09-16 18:00:43 +00:00
#!/usr/bin/env bash
2022-09-16 18:20:59 +00:00
#SBATCH -J RainEmbd
2022-09-16 18:00:43 +00:00
#SBATCH -N 1
#SBATCH -n 14
#SBATCH --gres=gpu:1
#SBATCH -o %j.%N.%a.out.log
#SBATCH -e %j.%N.%a.err.log
2022-09-23 18:21:57 +00:00
#SBATCH -p gpu05,gpu
2022-09-16 18:00:43 +00:00
#SBATCH --time=5-00:00:00
module load utilities/multi
module load readline/7.0
module load gcc/10.2.0
module load cuda/11.5.0
module load python/anaconda/4.6/miniconda/3.7
show_help() {
echo -e "Usage:" >&2;
echo -e " [INPUT=\"\$HOME/rainfallwater_records_embed.jsonl.gz\"] [POSTFIX=\"some_string\"] sbatch slurm-pretrain-plot.job" >&2;
echo -e "" >&2;
echo -e "....where:" >&2;
echo -e " INPUT The path to the input file (.jsonl.gz) containing the embedded data to plot (see the pretrain-predict subcommand for embedding data)" >&2;
2022-09-27 17:58:02 +00:00
echo -e " OUTPUT Optional. The filepath/dir to write output to." >&2;
2022-09-16 18:00:43 +00:00
echo -e " CHECKPOINT The filepath to the checkpoint (.hdf5) file to load" >&2;
echo -e " POSTFIX Arbitrary string to add to filename." >&2;
2022-09-27 17:41:35 +00:00
echo -e " ARGS Any additional args to pass to pretrain-plot." >&2;
2022-09-16 18:00:43 +00:00
echo -e "" >&2;
echo -e "The code used to identify the run is taken automatically from the filename of the config file." >&2;
exit;
}
INPUT="${INPUT:-$HOME/rainfallwater_records_tfrecord}";
CHECKPOINT="${CHECKPOINT:-output/2022-09-07_pretrain_contrast_dim1024/checkpoints/checkpoint_weights_e32_loss0.693.hdf5}"
if [[ -z "${INPUT}" ]]; then
echo -e "Error: No INPUT environment variable specified.\n" >&2;
show_help;
exit 0;
fi
if [[ -z "${CHECKPOINT}" ]]; then
echo -e "Error: No CHECKPOINT environment variable specified\n" >&2;
show_help;
exit 0;
fi
if [[ ! -d "${INPUT}" ]]; then
echo -e "Error: The input filepath at '${INPUT}' containing the input .tfrecord dataset either doesn't exist or isn't a directory.";
show_help;
exit 1;
fi
CODE="_contrast_embed_umap";
if [[ -n "${POSTFIX}" ]]; then
echo -e ">>> Applying postfix of ${POSTFIX}" >&2;
CODE="${CODE}_${POSTFIX}";
fi
2022-09-27 17:58:02 +00:00
if [[ -n "${OUTPUT}" ]]; then
filepath_output="${OUTPUT}";
else
filepath_output="output/rainfallwater_records_embed_$(date -u --rfc-3339=date)${CODE}.jsonl.gz";
fi
2022-09-16 18:00:43 +00:00
2022-09-27 18:05:54 +00:00
echo -e ">>> Input dirpath: ${INPUT}" >&2;
echo -e ">>> Output path: ${filepath_output}" >&2;
echo -e ">>> Postfix: ${POSTFIX}" >&2;
echo -e ">>> Checkpoint: ${CHECKPOINT}" >&2;
echo -e ">>> Code: ${CODE}" >&2;
echo -e ">>> Additional args: ${ARGS}";
2022-09-16 18:00:43 +00:00
export PATH=$HOME/software/bin:$PATH;
echo ">>> Installing requirements";
conda run -n py38 pip install -r requirements.txt;
2022-09-27 17:51:58 +00:00
echo ">>> Making predictions";
2022-09-16 18:00:43 +00:00
#shellcheck disable=SC2086
2022-09-27 18:06:41 +00:00
echo COMMAND: conda run -n py38 /usr/bin/env time -v src/index.py pretrain-predict --input "${INPUT}" --output "${filepath_output}" -c "${CHECKPOINT}" ${ARGS}
2022-09-27 17:41:35 +00:00
conda run -n py38 /usr/bin/env time -v src/index.py pretrain-predict --input "${INPUT}" --output "${filepath_output}" -c "${CHECKPOINT}" ${ARGS}
2022-09-16 18:00:43 +00:00
echo ">>> exited with code $?";
2022-09-16 18:36:57 +00:00
if [[ "${filepath_output}" != *.tfrecord* ]]; then
# dirpath_output="$(dirname "${filepath_output}")";
INPUT="${filepath_output}" POSTFIX="${POSTFIX}" sbatch slurm-pretrain-plot.job
echo ">>> No tfrecord output detected, queued UMAP plot"
fi