From f8a1e1b594b8b02a865a7efda13841bb33f4ae9c Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Thu, 19 Dec 2024 15:39:09 +0000 Subject: [PATCH] Fix lots of ruff linter warnings --- aimodel/src/lib/ai/RainfallWaterContraster.py | 10 +++++----- aimodel/src/lib/ai/RainfallWaterMono.py | 11 +++++------ aimodel/src/lib/ai/RainfallWaterSegmenter.py | 11 +++++------ .../src/lib/ai/components/CallbackExtraValidation.py | 2 +- aimodel/src/lib/ai/components/LossCrossEntropyDice.py | 1 - aimodel/src/lib/ai/components/LossDice.py | 4 +--- aimodel/src/lib/ai/components/MetricDice.py | 2 -- aimodel/src/lib/ai/components/MetricMeanIoU.py | 2 -- aimodel/src/lib/ai/components/MetricSensitivity.py | 2 -- aimodel/src/lib/ai/components/MetricSpecificity.py | 2 -- aimodel/src/lib/ai/model_rainfallwater_contrastive.py | 3 --- aimodel/src/lib/ai/model_rainfallwater_mono.py | 2 -- .../src/lib/ai/model_rainfallwater_segmentation.py | 2 -- aimodel/src/lib/dataset/dataset.py | 4 +--- aimodel/src/lib/dataset/dataset_encoderonly.py | 5 ++--- aimodel/src/lib/dataset/dataset_segmenter.py | 4 +--- aimodel/src/lib/primitives/env.py | 8 ++++---- aimodel/src/lib/vis/embeddings.py | 8 ++++---- aimodel/src/lib/vis/segmentation_plot.py | 4 ++-- 19 files changed, 31 insertions(+), 56 deletions(-) diff --git a/aimodel/src/lib/ai/RainfallWaterContraster.py b/aimodel/src/lib/ai/RainfallWaterContraster.py index 154c5bf..2af6add 100644 --- a/aimodel/src/lib/ai/RainfallWaterContraster.py +++ b/aimodel/src/lib/ai/RainfallWaterContraster.py @@ -6,8 +6,8 @@ import tensorflow as tf from ..dataset.batched_iterator import batched_iterator -from ..io.find_paramsjson import find_paramsjson -from ..io.readfile import readfile +# from ..io.find_paramsjson import find_paramsjson +# from ..io.readfile import readfile from ..io.writefile import writefile from .model_rainfallwater_contrastive import model_rainfallwater_contrastive @@ -16,7 +16,7 @@ from .helpers import summarywriter from .components.LayerContrastiveEncoder import LayerContrastiveEncoder from .components.LayerConvNeXtGamma import LayerConvNeXtGamma from .components.LayerCheeseMultipleOut import LayerCheeseMultipleOut -from .helpers.summarywriter import summarywriter + class RainfallWaterContraster(object): def __init__(self, dir_output=None, filepath_checkpoint=None, epochs=50, batch_size=64, **kwargs): @@ -28,8 +28,8 @@ class RainfallWaterContraster(object): self.batch_size = batch_size - if filepath_checkpoint == None: - if self.dir_output == None: + if filepath_checkpoint is None: + if self.dir_output is None: raise Exception("Error: dir_output was not specified, and since no checkpoint was loaded training mode is activated.") if not os.path.exists(self.dir_output): os.mkdir(self.dir_output) diff --git a/aimodel/src/lib/ai/RainfallWaterMono.py b/aimodel/src/lib/ai/RainfallWaterMono.py index 873b3d6..651ea01 100644 --- a/aimodel/src/lib/ai/RainfallWaterMono.py +++ b/aimodel/src/lib/ai/RainfallWaterMono.py @@ -4,10 +4,10 @@ import json from loguru import logger import tensorflow as tf -from ..dataset.batched_iterator import batched_iterator +# from ..dataset.batched_iterator import batched_iterator -from ..io.find_paramsjson import find_paramsjson -from ..io.readfile import readfile +# from ..io.find_paramsjson import find_paramsjson +# from ..io.readfile import readfile from ..io.writefile import writefile from .model_rainfallwater_mono import model_rainfallwater_mono @@ -16,7 +16,6 @@ from .helpers import summarywriter from .components.LayerConvNeXtGamma import LayerConvNeXtGamma from .components.LayerStack2Image import LayerStack2Image from .components.LossCrossentropy import LossCrossentropy -from .helpers.summarywriter import summarywriter class RainfallWaterMono(object): def __init__(self, dir_output=None, filepath_checkpoint=None, epochs=50, batch_size=64, **kwargs): @@ -28,8 +27,8 @@ class RainfallWaterMono(object): self.batch_size = batch_size - if filepath_checkpoint == None: - if self.dir_output == None: + if filepath_checkpoint is None: + if self.dir_output is None: raise Exception("Error: dir_output was not specified, and since no checkpoint was loaded training mode is activated.") if not os.path.exists(self.dir_output): os.mkdir(self.dir_output) diff --git a/aimodel/src/lib/ai/RainfallWaterSegmenter.py b/aimodel/src/lib/ai/RainfallWaterSegmenter.py index dee79cd..fed9200 100644 --- a/aimodel/src/lib/ai/RainfallWaterSegmenter.py +++ b/aimodel/src/lib/ai/RainfallWaterSegmenter.py @@ -4,10 +4,10 @@ import json from loguru import logger import tensorflow as tf -from ..dataset.batched_iterator import batched_iterator +# from ..dataset.batched_iterator import batched_iterator -from ..io.find_paramsjson import find_paramsjson -from ..io.readfile import readfile +# from ..io.find_paramsjson import find_paramsjson +# from ..io.readfile import readfile from ..io.writefile import writefile from .model_rainfallwater_segmentation import model_rainfallwater_segmentation @@ -15,7 +15,6 @@ from .helpers import make_callbacks from .helpers import summarywriter from .components.LayerConvNeXtGamma import LayerConvNeXtGamma from .components.LayerStack2Image import LayerStack2Image -from .helpers.summarywriter import summarywriter class RainfallWaterSegmenter(object): def __init__(self, dir_output=None, filepath_checkpoint=None, epochs=50, batch_size=64, **kwargs): @@ -27,8 +26,8 @@ class RainfallWaterSegmenter(object): self.batch_size = batch_size - if filepath_checkpoint == None: - if self.dir_output == None: + if filepath_checkpoint is None: + if self.dir_output is None: raise Exception("Error: dir_output was not specified, and since no checkpoint was loaded training mode is activated.") if not os.path.exists(self.dir_output): os.mkdir(self.dir_output) diff --git a/aimodel/src/lib/ai/components/CallbackExtraValidation.py b/aimodel/src/lib/ai/components/CallbackExtraValidation.py index 4ea2f07..3046cfb 100644 --- a/aimodel/src/lib/ai/components/CallbackExtraValidation.py +++ b/aimodel/src/lib/ai/components/CallbackExtraValidation.py @@ -27,7 +27,7 @@ class CallbackExtraValidation(tf.keras.callbacks.Callback): self.verbose = verbose def on_epoch_end(self, epoch, logs=None): - if logs == None: + if logs is None: logger.warning( "[CallbackExtraValidation] logs is None! Can't do anything here.") return False diff --git a/aimodel/src/lib/ai/components/LossCrossEntropyDice.py b/aimodel/src/lib/ai/components/LossCrossEntropyDice.py index 4a8169a..bcedc93 100644 --- a/aimodel/src/lib/ai/components/LossCrossEntropyDice.py +++ b/aimodel/src/lib/ai/components/LossCrossEntropyDice.py @@ -1,4 +1,3 @@ -import math import tensorflow as tf diff --git a/aimodel/src/lib/ai/components/LossDice.py b/aimodel/src/lib/ai/components/LossDice.py index 0df4baa..44d109a 100644 --- a/aimodel/src/lib/ai/components/LossDice.py +++ b/aimodel/src/lib/ai/components/LossDice.py @@ -1,5 +1,3 @@ -import math - import tensorflow as tf def dice_coef(y_true, y_pred, smooth=100): @@ -27,7 +25,7 @@ def dice_coef_loss(y_true, y_pred, **kwargs): y_true (Tensor): The ground truth y_pred (Tensor): The predicted output. Returns: - Tensor: The Dice coefficient, but as a loss value that decreases instead fo increases as the model learns. + Tensor: The Dice coefficient, but as a loss value that decreases instead of increases as the model learns. """ return 1 - dice_coef(y_true, y_pred, **kwargs) diff --git a/aimodel/src/lib/ai/components/MetricDice.py b/aimodel/src/lib/ai/components/MetricDice.py index 956f9f5..e1028df 100644 --- a/aimodel/src/lib/ai/components/MetricDice.py +++ b/aimodel/src/lib/ai/components/MetricDice.py @@ -1,5 +1,3 @@ -import math - import tensorflow as tf diff --git a/aimodel/src/lib/ai/components/MetricMeanIoU.py b/aimodel/src/lib/ai/components/MetricMeanIoU.py index 6469472..851831c 100644 --- a/aimodel/src/lib/ai/components/MetricMeanIoU.py +++ b/aimodel/src/lib/ai/components/MetricMeanIoU.py @@ -1,5 +1,3 @@ -import math - import tensorflow as tf diff --git a/aimodel/src/lib/ai/components/MetricSensitivity.py b/aimodel/src/lib/ai/components/MetricSensitivity.py index 80ec45e..3c1200b 100644 --- a/aimodel/src/lib/ai/components/MetricSensitivity.py +++ b/aimodel/src/lib/ai/components/MetricSensitivity.py @@ -1,5 +1,3 @@ -import math - import tensorflow as tf diff --git a/aimodel/src/lib/ai/components/MetricSpecificity.py b/aimodel/src/lib/ai/components/MetricSpecificity.py index 614db0f..4a0853b 100644 --- a/aimodel/src/lib/ai/components/MetricSpecificity.py +++ b/aimodel/src/lib/ai/components/MetricSpecificity.py @@ -1,5 +1,3 @@ -import math - import tensorflow as tf diff --git a/aimodel/src/lib/ai/model_rainfallwater_contrastive.py b/aimodel/src/lib/ai/model_rainfallwater_contrastive.py index 53911e0..67a2698 100644 --- a/aimodel/src/lib/ai/model_rainfallwater_contrastive.py +++ b/aimodel/src/lib/ai/model_rainfallwater_contrastive.py @@ -1,6 +1,3 @@ -import math - -from curses import meta from loguru import logger import tensorflow as tf diff --git a/aimodel/src/lib/ai/model_rainfallwater_mono.py b/aimodel/src/lib/ai/model_rainfallwater_mono.py index 843c63e..7c6a534 100644 --- a/aimodel/src/lib/ai/model_rainfallwater_mono.py +++ b/aimodel/src/lib/ai/model_rainfallwater_mono.py @@ -1,5 +1,3 @@ -import math - from loguru import logger import tensorflow as tf diff --git a/aimodel/src/lib/ai/model_rainfallwater_segmentation.py b/aimodel/src/lib/ai/model_rainfallwater_segmentation.py index 99ab884..8a37678 100644 --- a/aimodel/src/lib/ai/model_rainfallwater_segmentation.py +++ b/aimodel/src/lib/ai/model_rainfallwater_segmentation.py @@ -1,5 +1,3 @@ -import math - from loguru import logger import tensorflow as tf diff --git a/aimodel/src/lib/dataset/dataset.py b/aimodel/src/lib/dataset/dataset.py index cf85649..878c202 100644 --- a/aimodel/src/lib/dataset/dataset.py +++ b/aimodel/src/lib/dataset/dataset.py @@ -1,6 +1,5 @@ import os import math -import json from loguru import logger @@ -8,7 +7,6 @@ import tensorflow as tf from lib.dataset.read_metadata import read_metadata -from ..io.readfile import readfile from .primitives.shuffle import shuffle @@ -63,7 +61,7 @@ def make_dataset(filepaths, metadata, shape_water_desired=[100,100], compression dataset = dataset.shuffle(shuffle_buffer_size) dataset = dataset.map(parse_item(metadata, shape_water_desired=shape_water_desired, dummy_label=dummy_label), num_parallel_calls=tf.data.AUTOTUNE) - if batch_size != None: + if batch_size is not None: dataset = dataset.batch(batch_size, drop_remainder=True) if prefetch: dataset = dataset.prefetch(0 if "NO_PREFETCH" in os.environ else tf.data.AUTOTUNE) diff --git a/aimodel/src/lib/dataset/dataset_encoderonly.py b/aimodel/src/lib/dataset/dataset_encoderonly.py index a65c0e8..0ba056e 100644 --- a/aimodel/src/lib/dataset/dataset_encoderonly.py +++ b/aimodel/src/lib/dataset/dataset_encoderonly.py @@ -1,6 +1,5 @@ import os import math -import json from loguru import logger @@ -8,9 +7,9 @@ import tensorflow as tf from lib.dataset.read_metadata import read_metadata -from ..io.readfile import readfile from .primitives.shuffle import shuffle from .parse_heightmap import parse_heightmap +from .dataset_mono import dataset_mono @@ -125,7 +124,7 @@ def make_dataset(filepaths, compression_type="GZIP", parallel_reads_multiplier=3 # defaults = (33*33 + 1) * 2**16 * 8 = about 2.219GiB dataset = dataset.shuffle(shuffle_buffer_size) - if batch_size != None: + if batch_size is not None: dataset = dataset.batch(batch_size, drop_remainder=True) if prefetch: dataset = dataset.prefetch(0 if "NO_PREFETCH" in os.environ else tf.data.AUTOTUNE) diff --git a/aimodel/src/lib/dataset/dataset_segmenter.py b/aimodel/src/lib/dataset/dataset_segmenter.py index cb78b96..63d03cc 100644 --- a/aimodel/src/lib/dataset/dataset_segmenter.py +++ b/aimodel/src/lib/dataset/dataset_segmenter.py @@ -1,12 +1,10 @@ import os import math -import json from loguru import logger import tensorflow as tf from lib.dataset.read_metadata import read_metadata -from ..io.readfile import readfile from .primitives.shuffle import shuffle @@ -62,7 +60,7 @@ def make_dataset(filepaths, metadata, shape_water_desired=[100,100], water_thres dataset = dataset.shuffle(shuffle_buffer_size) dataset = dataset.map(parse_item(metadata, shape_water_desired=shape_water_desired, water_threshold=water_threshold), num_parallel_calls=tf.data.AUTOTUNE) - if batch_size != None: + if batch_size is not None: dataset = dataset.batch(batch_size, drop_remainder=True) if prefetch: dataset = dataset.prefetch(0 if "NO_PREFETCH" in os.environ else tf.data.AUTOTUNE) diff --git a/aimodel/src/lib/primitives/env.py b/aimodel/src/lib/primitives/env.py index c90928e..d86d1ba 100644 --- a/aimodel/src/lib/primitives/env.py +++ b/aimodel/src/lib/primitives/env.py @@ -51,7 +51,7 @@ def read(name, type_class, default=SYM_RAISE_EXCEPTION): """ if name not in os.environ: - if type_class == bool and default == SYM_RAISE_EXCEPTION: + if type_class is bool and default == SYM_RAISE_EXCEPTION: default = False if default == SYM_RAISE_EXCEPTION: raise Exception(f"Error: Environment variable {name} does not exist") @@ -59,8 +59,8 @@ def read(name, type_class, default=SYM_RAISE_EXCEPTION): return default result = os.environ[name] - if type_class == bool: - result = False if default == True else True + if type_class is bool: + result = False if default is True else True else: result = type_class(result) @@ -99,7 +99,7 @@ def print_all(table=True): for env in envs_read: key, value, is_default = env prefix = "* " if is_default else "" - print(f"> {key.ljust(width_name)} {value}") + print(f"> {prefix}{key.ljust(width_name)} {value}") print(f"Total {len(envs_read)} values") print("===================================") diff --git a/aimodel/src/lib/vis/embeddings.py b/aimodel/src/lib/vis/embeddings.py index 0df6aa2..8199075 100644 --- a/aimodel/src/lib/vis/embeddings.py +++ b/aimodel/src/lib/vis/embeddings.py @@ -1,8 +1,8 @@ -import os +# import os import umap import umap.plot -import numpy as np +# import numpy as np import matplotlib.pylab as plt import pandas @@ -24,7 +24,7 @@ def vis_embeddings(filepath_output, features): umap.plot.points(dimreducer, ax=axes["A"] ) - axes["A"].set_title(f"UMAP Dimensionality Reduction", fontsize=20) + axes["A"].set_title("UMAP Dimensionality Reduction", fontsize=20) # 2: Parallel coordinates dataframe = pandas.DataFrame(features) @@ -39,7 +39,7 @@ def vis_embeddings(filepath_output, features): sort_labels=True ) - axes["B"].set_title(f"Parallel coordinates plot", fontsize=20) + axes["B"].set_title("Parallel coordinates plot", fontsize=20) plt.suptitle(f"RainfallContrastive embeddings | rainfall | E2 ConvNeXt | {len(features)} items", fontsize=28, weight="bold") plt.savefig(filepath_output) diff --git a/aimodel/src/lib/vis/segmentation_plot.py b/aimodel/src/lib/vis/segmentation_plot.py index aca3165..67b87da 100644 --- a/aimodel/src/lib/vis/segmentation_plot.py +++ b/aimodel/src/lib/vis/segmentation_plot.py @@ -17,11 +17,11 @@ def segmentation_plot(water_actual, water_predict, model_code, filepath_output): figure, axes = plt.subplot_mosaic("AB", figsize=(width*px, height*px)) axes["A"].imshow(water_actual) - axes["A"].set_title(f"Actual", fontsize=20) + axes["A"].set_title("Actual", fontsize=20) axes["B"].imshow(water_predict) - axes["B"].set_title(f"Predicted", fontsize=20) + axes["B"].set_title("Predicted", fontsize=20) plt.suptitle(f"Rainfall → Water depth prediction | {model_code}", fontsize=28, weight="bold")