sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-25 10:32:59 +00:00
Code Issues Projects Releases Packages Wiki Activity

dlr eo: add STEPS_PER_EXECUTION

Browse source
This commit is contained in:
Starbeamrainbowlabs 2023-02-02 15:47:08 +00:00
parent f8202851a1
commit f7666865a0
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
2 changed files with 8 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
aimodel/slurm-encoderonly-rainfall.job
View file

@ -34,6 +34,7 @@ show_help() {
echo -e " CHANNELS=8 The number of channels the input data has." >&2; echo -e " CHANNELS=8 The number of channels the input data has." >&2;
echo -e " WINDOW_SIZE=33 The window size to use when convolving the input dataset for single pixel prediction." >&2; echo -e " WINDOW_SIZE=33 The window size to use when convolving the input dataset for single pixel prediction." >&2;
echo -e " STEPS_PER_EPOCH The number of steps to consider an epoch. Defaults to None, which means use the entire dataset." >&2; echo -e " STEPS_PER_EPOCH The number of steps to consider an epoch. Defaults to None, which means use the entire dataset." >&2;
echo -e " STEPS_PER_EXECUTION The number of steps to do before returning to do callbacks. High numbers boost performance. Defaults to 1." >&2;
echo -e " EPOCHS=25 The number of epochs to train for." >&2; echo -e " EPOCHS=25 The number of epochs to train for." >&2;
echo -e " LEARNING_RATE The learning rate to use. Default: 0.001." >&2; echo -e " LEARNING_RATE The learning rate to use. Default: 0.001." >&2;
# echo -e " NO_REMOVE_ISOLATED_PIXELS Set to any value to avoid the engine from removing isolated pixels - that is, water pixels with no other surrounding pixels, either side to side to diagonally." >&2; # echo -e " NO_REMOVE_ISOLATED_PIXELS Set to any value to avoid the engine from removing isolated pixels - that is, water pixels with no other surrounding pixels, either side to side to diagonally." >&2;
@ -66,7 +67,7 @@ echo -e ">>> DIR_OUTPUT: ${DIR_OUTPUT}";
echo -e ">>> Additional args: ${ARGS}"; echo -e ">>> Additional args: ${ARGS}";
export PATH=$HOME/software/bin:$PATH; export PATH=$HOME/software/bin:$PATH;
export BATCH_SIZE DIRPATH_RAINFALLWATER PATH_HEIGHTMAP STEPS_PER_EPOCH DIRPATH_OUTPUT PATH_CHECKPOINT CHANNELS WINDOW_SIZE EPOCHS LEARNING_RATE; export BATCH_SIZE DIRPATH_RAINFALLWATER PATH_HEIGHTMAP STEPS_PER_EPOCH DIRPATH_OUTPUT PATH_CHECKPOINT CHANNELS WINDOW_SIZE EPOCHS LEARNING_RATE STEPS_PER_EXECUTION;
#LOSS ; #LOSS ;
echo ">>> Installing requirements"; echo ">>> Installing requirements";

9
aimodel/src/encoderonly_test_rainfall.py
View file

@ -26,6 +26,7 @@ EPOCHS = int(os.environ["EPOCHS"]) if "EPOCHS" in os.environ else 25
BATCH_SIZE = int(os.environ["BATCH_SIZE"]) if "BATCH_SIZE" in os.environ else 64 BATCH_SIZE = int(os.environ["BATCH_SIZE"]) if "BATCH_SIZE" in os.environ else 64
WINDOW_SIZE = int(os.environ["WINDOW_SIZE"]) if "WINDOW_SIZE" in os.environ else 33 WINDOW_SIZE = int(os.environ["WINDOW_SIZE"]) if "WINDOW_SIZE" in os.environ else 33
STEPS_PER_EPOCH = int(os.environ["STEPS_PER_EPOCH"]) if "STEPS_PER_EPOCH" in os.environ else None STEPS_PER_EPOCH = int(os.environ["STEPS_PER_EPOCH"]) if "STEPS_PER_EPOCH" in os.environ else None
STEPS_PER_EXECUTION = int(os.environ["STEPS_PER_EXECUTION"]) if "STEPS_PER_EXECUTION" in os.environ else None
LEARNING_RATE = float(os.environ["LEARNING_RATE"]) if "LEARNING_RATE" in os.environ else 0.001 LEARNING_RATE = float(os.environ["LEARNING_RATE"]) if "LEARNING_RATE" in os.environ else 0.001
logger.info("Encoder-only rainfall radar TEST") logger.info("Encoder-only rainfall radar TEST")
@ -64,7 +65,7 @@ dataset_train, dataset_validate = dataset_encoderonly(
# ██ ██ ██ ██ ██ ██ ██ ██ ██ # ██ ██ ██ ██ ██ ██ ██ ██ ██
# ██ ██ ██████ ██████ ███████ ███████ # ██ ██ ██████ ██████ ███████ ███████
def make_encoderonly(windowsize, channels, encoder="convnext", water_bins=2, **kwargs): def make_encoderonly(windowsize, channels, encoder="convnext", water_bins=2, steps_per_execution=1, **kwargs):
if encoder == "convnext": if encoder == "convnext":
model = make_convnext( model = make_convnext(
input_shape=(windowsize, windowsize, channels), input_shape=(windowsize, windowsize, channels),
@ -95,7 +96,8 @@ def make_encoderonly(windowsize, channels, encoder="convnext", water_bins=2, **k
loss = tf.keras.losses.SparseCategoricalCrossentropy(), loss = tf.keras.losses.SparseCategoricalCrossentropy(),
metrics = [ metrics = [
tf.keras.metrics.SparseCategoricalAccuracy() tf.keras.metrics.SparseCategoricalAccuracy()
] ],
steps_per_execution=steps_per_execution
) )
return model return model
@ -103,7 +105,8 @@ def make_encoderonly(windowsize, channels, encoder="convnext", water_bins=2, **k
model = make_encoderonly( model = make_encoderonly(
windowsize=WINDOW_SIZE, windowsize=WINDOW_SIZE,
channels=CHANNELS channels=CHANNELS.
steps_per_execution=STEPS_PER_EXECUTION
) )
summarywriter(model, os.path.join(DIRPATH_OUTPUT, "summary.txt")) summarywriter(model, os.path.join(DIRPATH_OUTPUT, "summary.txt"))