sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-21 17:03:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

slurm/dlr: logging

Browse source 
it Should™ work now?
TODO test this!
This commit is contained in:
Starbeamrainbowlabs 2024-11-14 19:55:59 +00:00
parent c82e4bf5ac
commit 8befef5fc1
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
aimodel/slurm-TEST-deeplabv3p-rainfall.job
View file

@ -30,12 +30,10 @@ if command_exists module; then
module load python/anaconda/4.6/miniconda/3.7 module load python/anaconda/4.6/miniconda/3.7
else else
echo "[bash/runner]: module command not present, not loading modules" >&2; echo "[slurm_runner]: module command not present, not loading modules" >&2;
fi fi
show_help() { show_help() {
echo -e "Trains a TEST DeepLabv3+ model using rainfall radar and water depth data." >&2; echo -e "Trains a TEST DeepLabv3+ model using rainfall radar and water depth data." >&2;
echo -e "" >&2; echo -e "" >&2;
@ -102,13 +100,15 @@ echo ">>> Installing requirements";
if [[ -n "${USE_CONDA}" ]]; then if [[ -n "${USE_CONDA}" ]]; then
conda run -n py38 pip install -q -r requirements.txt; conda run -n py38 pip install -q -r requirements.txt;
else else
echo "[bash/runner]: USE_CONDA env var NOT specified, not installing pip packages." >&2; echo "[slurm_runner]: USE_CONDA env var NOT specified, not installing pip packages." >&2;
fi fi
echo ">>> Training model"; echo ">>> Training model";
#shellcheck disable=SC2016 #shellcheck disable=SC2016
if [[ -n "${USE_CONDA}" ]]; then if [[ -n "${USE_CONDA}" ]]; then
echo "[slurm_runner]: Running harness via conda as USE_CONDA env var was defined" >&2;
/usr/bin/env time -v conda run -n py38 bash -c 'src/deeplabv3_plus_test_rainfall.py >>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.out.log" 2>>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.err.log"; echo "[slurm_runner] EXIT_CODE: $?" >>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.out.log";'; /usr/bin/env time -v conda run -n py38 bash -c 'src/deeplabv3_plus_test_rainfall.py >>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.out.log" 2>>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.err.log"; echo "[slurm_runner] EXIT_CODE: $?" >>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.out.log";';
else else
echo "[slurm_runner]: Ignoring conda and running harness directly as USE_CONDA env var was NOT defined" >&2;
/usr/bin/env time -v src/deeplabv3_plus_test_rainfall.py >>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.out.log" 2>>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.err.log"; /usr/bin/env time -v src/deeplabv3_plus_test_rainfall.py >>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.out.log" 2>>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.err.log";
echo "[slurm_runner] EXIT_CODE: $?" >>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.out.log"; echo "[slurm_runner] EXIT_CODE: $?" >>"${DIR_OUTPUT}/experiment.${SLURM_JOB_ID}.out.log";
fi fi