core\feat: poetry dependencies

- locked dependencies with poetry-
- increase mypy strictness
- type corrections across all submodules

.github/workflows/python-ci.yml

@@ -70,12 +70,12 @@ jobs:
       - name: Run mypy
         run: |
           cd core
-          python -m mypy .
+          python -m mypy seg_tgce
 
       - name: Run pylint
         run: |
           cd core
-          python -m pylint .
+          python -m pylint seg_tgce
 
   lint-docs:
     runs-on: ubuntu-22.04

core/.pylintrc

@@ -1,3 +1,4 @@
 [MASTER]
 disable = missing-module-docstring,missing-function-docstring,missing-class-docstring,redefined-outer-name
+ignored-modules = keras
 init-hook='import sys; sys.path.append("core")'

core/mypy.ini

@@ -1,3 +1,5 @@
 [mypy]
 python_version = 3.11
-disable_error_code = import-untyped
+disable_error_code = import-untyped
+disallow_untyped_calls = True
+disallow_incomplete_defs = True

core/pyproject.toml

@@ -4,7 +4,6 @@ version = "0.0.1"
 authors = ["Brandon Lotero <[email protected]>"]
 description = "A package for the SEG TGCE project"
 readme = "README.md"
-requires-python = ">=3.11"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
@@ -15,6 +14,24 @@ repository = "https://github.com/blotero/seg_tgce"
 
 [tool.poetry.dependencies]
 python = "^3.11"
+numpy = "^1.26.4"
+keras = "^3.3.3"
+tensorflow = "^2.16.1"
+matplotlib = "^3.8.4"
+opencv-python = "^4.9.0.80"
+gcpds-image-segmentation = { git = "https://github.com/UN-GCPDS/python-gcpds.image_segmentation" }
+tensorflow-datasets = "^4.9.4"
+
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.2.0"
+
+
+[tool.poetry.group.dev.dependencies]
+mypy = "^1.10.0"
+pylint = "^3.1.0"
+black = "^24.4.2"
+isort = "^5.13.2"
 
 [build-system]
 requires = ["poetry-core"]

core/seg_tgce/data/crowd_seg/generator.py

@@ -1,24 +1,25 @@
 import logging
 import os
-from typing import List
+from typing import List, Tuple
 
 import numpy as np
 from keras.preprocessing.image import img_to_array, load_img
 from keras.utils import Sequence
 from matplotlib import pyplot as plt
 
 LOGGER = logging.getLogger(__name__)
+logging.basicConfig(level=logging.WARNING)
 
 
 class ImageDataGenerator(Sequence):  # pylint: disable=too-many-instance-attributes
     def __init__(  # pylint: disable=too-many-arguments
         self,
-        image_dir,
-        mask_dir,
-        n_classes,
-        image_size=(256, 256),
-        batch_size=32,
-        shuffle=True,
+        image_dir: str,
+        mask_dir: str,
+        n_classes: int,
+        image_size: Tuple[int, int] = (256, 256),
+        batch_size: int = 32,
+        shuffle: bool = True,
     ):
         self.image_dir = image_dir
         self.mask_dir = mask_dir
@@ -48,16 +49,16 @@ def __getitem__(self, index):
         images, masks = self.__data_generation(batch_filenames)
         return images, masks
 
-    def on_epoch_end(self):
+    def on_epoch_end(self) -> None:
         if self.shuffle:
             np.random.shuffle(self.image_filenames)
 
     def visualize_sample(
         self,
         scorers: List[str],
-        batch_index=1,
-        sample_index=1,
-    ):
+        batch_index: int = 1,
+        sample_index: int = 1,
+    ) -> plt.Figure:
         images, masks = self[batch_index]
 
         fig, axes = plt.subplots(len(scorers), self.n_classes + 1)

core/seg_tgce/data/crowd_seg/visualizer.py

@@ -15,7 +15,7 @@ def visualize_data(  # pylint: disable=too-many-locals
     labelers: Tuple[str, str],
     base_path: str,
     save_path: str,
-):
+) -> None:
     """
     Simple routine for visualizing some patches and masks
     """

core/seg_tgce/data/oxford_pet/disturbance/model.py

@@ -73,7 +73,7 @@ def download_base_model() -> str:
         if file.endswith(model_extension):
             paths.append(file)
 
-    return paths[0]
+    return os.path.abspath(paths[0])
 
 
 def find_last_encoder_conv_layer(model: Model) -> Layer:

core/seg_tgce/data/oxford_pet/oxford_pet.py

@@ -2,7 +2,7 @@
 from gcpds.image_segmentation.datasets.segmentation import OxfordIiitPet
 from keras.models import Model
 
-from core.seg_tgce.data.utils import map_dataset_multiple_annotators
+from seg_tgce.data.utils import map_dataset_multiple_annotators
 
 MODEL_ORIGINAL_SHAPE = (256, 256)
 

core/seg_tgce/data/utils.py

@@ -10,7 +10,7 @@ def disturb_mask(
     image: tf.Tensor,
     model_shape: Tuple[int, int],
     target_shape: Tuple[int, int],
-):
+) -> tf.Tensor:
     return tf.image.resize(model(tf.image.resize(image, model_shape)), target_shape)
 
 

core/seg_tgce/experiments/apr_14_2024/experiment.py

@@ -1,8 +1,9 @@
 import asyncio
 import signal
+from types import FrameType
 
-from core.seg_tgce.run.oxford_ma_runner.runner import OxfordMARunner
-from core.seg_tgce.run.runner import RunningSessionParams
+from seg_tgce.run.oxford_ma_runner.runner import OxfordMARunner
+from seg_tgce.run.runner import RunningSessionParams
 
 runner = OxfordMARunner(
     params=RunningSessionParams(
@@ -18,12 +19,15 @@
 )
 
 
-async def interruption_handler():
+async def interruption_handler(  # pylint: disable=unused-argument
+    signum: int,
+    fram: FrameType | None,
+) -> None:
     partial_res = await runner.stop()
     print(partial_res)
 
 
-async def main():
+async def main() -> None:
     signal.signal(signal.SIGINT, interruption_handler)
     results = await runner.run()
     print(results)

core/seg_tgce/loss/tgce.py

@@ -17,7 +17,7 @@ class TcgeConfig:
     gamma: float = 0.1
 
 
-def binary_entropy(target, pred):
+def binary_entropy(target: tf.Tensor, pred: tf.Tensor) -> tf.Tensor:
     """
     Adds binary entropy to the loss.
     """

core/seg_tgce/metrics/dice_coefficient.py

@@ -1,19 +1,25 @@
 import keras.backend as K
 from keras.losses import Loss
 from keras.saving import get_custom_objects, register_keras_serializable
-from tensorflow import gather
+from tensorflow import Tensor, gather
 
 get_custom_objects().clear()
 
 
 @register_keras_serializable(package="MyLayers")
 class DiceCoefficient(Loss):
-    def __init__(self, smooth=1.0, target_class=None, name="DiceCoefficient", **kwargs):
+    def __init__(  # type: ignore
+        self,
+        smooth: float = 1.0,
+        target_class=int | None,
+        name: str = "DiceCoefficient",
+        **kwargs
+    ):
         self.smooth = smooth
         self.target_class = target_class
         super().__init__(name=name, **kwargs)
 
-    def call(self, y_true, y_pred):
+    def call(self, y_true: Tensor, y_pred: Tensor) -> Tensor:
         intersection = K.sum(y_true * y_pred, axis=[1, 2])
         union = K.sum(y_true, axis=[1, 2]) + K.sum(y_pred, axis=[1, 2])
         dice_coef = -(2.0 * intersection + self.smooth) / (union + self.smooth)
@@ -27,6 +33,6 @@ def call(self, y_true, y_pred):
 
     def get_config(
         self,
-    ):
+    ) -> dict:
         base_config = super().get_config()
         return {**base_config, "smooth": self.smooth, "target_class": self.target_class}

core/seg_tgce/models/ma_model.py

@@ -2,7 +2,9 @@
 from tensorflow import GradientTape
 
 
-class ModelMultipleAnnotators(Model):  # pylint: disable=abstract-method
+class ModelMultipleAnnotators(
+    Model
+):  # pylint: disable=abstract-method, too-few-public-methods
     def train_step(self, data):
         x, y = data
 

core/seg_tgce/models/unet.py

@@ -1,4 +1,5 @@
 from functools import partial
+from typing import Tuple
 
 from keras.initializers import GlorotUniform
 from keras.layers import (
@@ -19,17 +20,17 @@
 upsample = partial(UpSampling2D, (2, 2))
 
 
-def kernel_initializer(seed):
+def kernel_initializer(seed: float) -> GlorotUniform:
     return GlorotUniform(seed=seed)
 
 
 def unet_tgce(  # pylint: disable=too-many-statements
-    input_shape=(128, 128, 3),
-    name="UNET",
-    out_channels=2,
-    n_scorers=5,
-    out_act_functions=("softmax", "sigmoid"),
-):
+    input_shape: Tuple[int, int, int] = (128, 128, 3),
+    name: str = "UNET",
+    out_channels: int = 2,
+    n_scorers: int = 5,
+    out_act_functions: Tuple[str, str] = ("softmax", "sigmoid"),
+) -> ModelMultipleAnnotators:
     # Encoder
     input_layer = Input(shape=input_shape)
 

core/seg_tgce/run/oxford_ma_runner/model_result.py

@@ -9,7 +9,7 @@ class ModelResult:
     val_losses: list[float]
     val_dices: list[float]
 
-    def save_as_json(self, filename):
+    def save_as_json(self, filename: str) -> None:
         dataclass_dict = self.__dict__
         json_str = json.dumps(dataclass_dict, indent=4)
         with open(filename, "w", encoding="utf-8") as f:

core/seg_tgce/run/oxford_ma_runner/plotting.py

@@ -1,10 +1,18 @@
+from typing import List
+
 import numpy as np
+from keras import Tensor
 from matplotlib import pyplot as plt
 
 
 def epoch_progress_plotter(  # pylint: disable=too-many-arguments
-    x, y, predictions, num_img, noise_values, num_annotators
-):
+    x: Tensor,
+    y: Tensor,
+    predictions: Tensor,
+    num_img: int,
+    noise_values: List[float],
+    num_annotators: int,
+) -> None:
     _, axes = plt.subplots(4, num_annotators, figsize=(15, 9))
 
     axes[0, 0].imshow(x[num_img, :, :, :])
@@ -57,7 +65,7 @@ def plot_losses_and_metrics(
     val_losses: list[float],
     val_dices: list[float],
     title: str,
-):
+) -> None:
     plt.figure(figsize=(12, 4))
     plt.subplot(1, 2, 1)
     plt.plot(losses, label="Training Loss")

core/seg_tgce/run/oxford_ma_runner/runner.py

@@ -1,31 +1,33 @@
+from typing import List
+
 import numpy as np
 import tensorflow as tf
-from keras.models import load_model
+from keras.models import Model, load_model
 from keras.optimizers import Adam
 
-from core.seg_tgce.data.oxford_pet.disturbance.model import (
+from seg_tgce.data.oxford_pet.disturbance.model import (
     download_base_model,
     find_last_encoder_conv_layer,
     produce_disturbed_models,
 )
-from core.seg_tgce.data.oxford_pet.oxford_pet import get_data_multiple_annotators
-from core.seg_tgce.loss.tgce import TcgeConfig, TcgeSs
-from core.seg_tgce.metrics.dice_coefficient import DiceCoefficient
-from core.seg_tgce.models.unet import unet_tgce
-from core.seg_tgce.run.oxford_ma_runner.model_result import ModelResult
-from core.seg_tgce.run.oxford_ma_runner.plotting import (
+from seg_tgce.data.oxford_pet.oxford_pet import get_data_multiple_annotators
+from seg_tgce.loss.tgce import TcgeConfig, TcgeSs
+from seg_tgce.metrics.dice_coefficient import DiceCoefficient
+from seg_tgce.models.unet import unet_tgce
+from seg_tgce.run.oxford_ma_runner.model_result import ModelResult
+from seg_tgce.run.oxford_ma_runner.plotting import (
     epoch_progress_plotter,
     plot_losses_and_metrics,
 )
-from core.seg_tgce.run.runner import (
+from seg_tgce.run.runner import (
     Runner,
     RunningSessionParams,
     SessionPartialResults,
     SessionResults,
 )
 
 
-def fetch_models(noise_levels_snr: list[float]):
+def fetch_models(noise_levels_snr: list[float]) -> List[Model]:
     model_path = download_base_model()
     model_ann = load_model(model_path, compile=False)
 
@@ -101,7 +103,9 @@ async def run(  # pylint: disable=too-many-locals, too-many-statements
                         loss = loss_fn.call(y_batch, predictions)
 
                     gradients = tape.gradient(loss, model.trainable_variables)
-                    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
+                    optimizer.apply_gradients(
+                        zip(gradients, model.trainable_variables)  # type:ignore
+                    )
 
                     epoch_loss += float(loss.numpy())
                     epoch_losses.append(loss.numpy())

core/tests/data/crowd_seg/test_map.py

@@ -3,7 +3,7 @@
 
 import pytest
 
-from core.seg_tgce.data.crowd_seg.map import DataTarget, find_annotators_alias
+from seg_tgce.data.crowd_seg.map import DataTarget, find_annotators_alias
 
 BASE_PATH = Path("/home/brandon/unal/maestria/datasets/Histology Data")
 

core/tests/data/test_disturbance.py

@@ -1,7 +1,7 @@
 import tensorflow as tf
 from keras.models import load_model
 
-from core.seg_tgce.data.oxford_pet.disturbance.model import (
+from seg_tgce.data.oxford_pet.disturbance.model import (
     download_base_model,
     find_last_encoder_conv_layer,
     produce_disturbed_models,

core/tests/data/test_oxford_pet.py

@@ -1,11 +1,11 @@
 from keras.models import load_model
 
-from core.seg_tgce.data.oxford_pet.disturbance.model import (
+from seg_tgce.data.oxford_pet.disturbance.model import (
     download_base_model,
     find_last_encoder_conv_layer,
     produce_disturbed_models,
 )
-from core.seg_tgce.data.oxford_pet.oxford_pet import get_data_multiple_annotators
+from seg_tgce.data.oxford_pet.oxford_pet import get_data_multiple_annotators
 
 
 def test_oxford_pet_flow() -> None: