Re-factor package structure and add tasks dependency extra

.github/workflows/pip_install.yml

@@ -0,0 +1,43 @@
+name: pip_install
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+jobs:
+
+  tests:
+    name: "Python ${{ matrix.python-version }}"
+    runs-on: ubuntu-20.04
+    timeout-minutes: 10
+
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+
+    steps:
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+          fetch-depth: 1
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install package (without extra)
+        run: python -m pip install -e .
+
+      - name: Test package import (without extra)
+        run: python -c 'import fractal_tasks_core; print("OK")'
+
+      - name: Install package (with fractal-tasks extra)
+        run: python -m pip install -e .[fractal-tasks]
+
+      - name: Test package import (with fractal-tasks extra)
+        run: python -c 'import fractal_tasks_core.tasks.cellpose_segmentation; print("OK")'

examples/tools/show_FOV_ROIs/MeasurementDetail_2x2_well.mrf

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<bts:MeasurementDetail bts:Version="1.0" bts:OperatorName="Joel" bts:Title="20200812-CardiomyocyteDifferentiation14-Cycle1" bts:Application="" bts:BeginTime="2020-08-12T17:27:17.603876+02:00" bts:EndTime="2020-08-13T03:20:17.1789113+02:00" bts:MeasurementSettingFileName="20200812-Joel-CardiomyocyteDifferentiation14-Cycle1.mes" bts:ColumnCount="12" bts:RowCount="8" bts:TimePointCount="5" bts:FieldCount="72" bts:ZCount="42" bts:TargetSystem="CV7000 FD" bts:ReleaseNumber="R1.17.05" xmlns:bts="http://www.yokogawa.co.jp/BTS/BTSSchema/1.0">
+  <bts:MeasurementSamplePlate bts:Name="20200812-CardiomyocyteDifferentiation14-Cycle1" bts:WellPlateFileName="20200812-CardiomyocyteDifferentiation14-Cycle1.wpi" bts:WellPlateProductFileName="1009602002_Greiner_#655090.wpp" />
+  <bts:MeasurementChannel bts:Ch="1" bts:HorizontalPixelDimension="0.1625" bts:VerticalPixelDimension="0.1625" bts:CameraNumber="3" bts:InputBitDepth="16" bts:InputLevel="1000" bts:HorizontalPixels="2560" bts:VerticalPixels="2160" bts:FilterWheelPosition="6" bts:FilterPosition="1" bts:ShadingCorrectionSource="SC_BP445-45_40x_M10_CH01.tif" />
+  <bts:MeasurementChannel bts:Ch="2" bts:HorizontalPixelDimension="0.1625" bts:VerticalPixelDimension="0.1625" bts:CameraNumber="1" bts:InputBitDepth="16" bts:InputLevel="1000" bts:HorizontalPixels="2560" bts:VerticalPixels="2160" bts:FilterWheelPosition="4" bts:FilterPosition="2" bts:ShadingCorrectionSource="SC_BP675-30_40x_M10_CH02.tif" />
+  <bts:MeasurementChannel bts:Ch="3" bts:HorizontalPixelDimension="0.1625" bts:VerticalPixelDimension="0.1625" bts:CameraNumber="2" bts:InputBitDepth="16" bts:InputLevel="2000" bts:HorizontalPixels="2560" bts:VerticalPixels="2160" bts:FilterWheelPosition="5" bts:FilterPosition="1" bts:ShadingCorrectionSource="SC_BP525-50_40x_M10_CH03.tif" />
+</bts:MeasurementDetail>

examples/tools/show_FOV_ROIs/README.md

@@ -0,0 +1 @@
+NOTE: running this script requires `matplotlib`

examples/tools/show_FOV_ROIs/show_FOV_ROIs.py

@@ -0,0 +1,83 @@
+"""
+An example of visualizing FOV ROIs and their overlaps.
+"""
+import matplotlib.pyplot as plt
+
+from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata
+from fractal_tasks_core.lib_ROI_overlaps import run_overlap_check
+
+
+def _plot_rectangle(min_x, min_y, max_x, max_y, overlapping):
+    x = [min_x, max_x, max_x, min_x, min_x]
+    y = [min_y, min_y, max_y, max_y, min_y]
+    if overlapping:
+        plt.plot(x, y, ",-", lw=2.5, zorder=2)
+    else:
+        plt.plot(x, y, ",-", lw=0.3, c="k", zorder=1)
+
+
+def _plotting_function(
+    xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well
+):
+    plt.figure()
+    num_lines = len(xmin)
+    for line in range(num_lines):
+        min_x, max_x = [a[line] for a in [xmin, xmax]]
+        min_y, max_y = [a[line] for a in [ymin, ymax]]
+
+        _plot_rectangle(
+            min_x, min_y, max_x, max_y, line in list_overlapping_FOVs
+        )
+        plt.text(
+            0.5 * (min_x + max_x),
+            0.5 * (min_y + max_y),
+            f"{line + 1}",
+            ha="center",
+            va="center",
+            fontsize=14,
+        )
+
+    plt.gca().set_aspect(1)
+    plt.xlabel("x (um)", fontsize=12)
+    plt.ylabel("y (um)", fontsize=12)
+    plt.title(f"Well {selected_well}")
+
+    plt.figure()
+    for line in range(num_lines):
+        min_x, max_x = [a[line] for a in [xmin, xmax]]
+        min_y, max_y = [a[line] for a in [ymin, ymax]]
+
+        _plot_rectangle(
+            min_x, min_y, max_x, max_y, line in list_overlapping_FOVs
+        )
+        plt.text(
+            0.5 * (min_x + max_x),
+            0.5 * (min_y + max_y),
+            f"{line + 1}",
+            ha="center",
+            va="center",
+            fontsize=14,
+        )
+
+    plt.gca().set_aspect(1)
+    plt.xlabel("x (um)", fontsize=12)
+    plt.ylabel("y (um)", fontsize=12)
+    plt.title(f"Well {selected_well}")
+
+
+if __name__ == "__main__":
+    mlf_path = "MeasurementData_2x2_well.mlf"
+    mrf_path = "MeasurementDetail_2x2_well.mrf"
+    site_metadata, total_files = parse_yokogawa_metadata(mrf_path, mlf_path)
+
+    plt.close()
+    run_overlap_check(
+        site_metadata, tol=0, plotting_function=_plotting_function
+    )
+    plt.savefig("fig_tol_0.pdf")
+
+    plt.close()
+    run_overlap_check(
+        site_metadata, tol=1e-10, plotting_function=_plotting_function
+    )
+    plt.savefig("fig_tol_1e-10.pdf")

fractal_tasks_core/__FRACTAL_MANIFEST__.json

@@ -76,7 +76,13 @@
         "title": "CreateOmeZarr",
         "type": "object"
       },
-      "executable": "create_ome_zarr.py",
+      "default_args": {
+        "coarsening_xy": 2,
+        "image_extension": "tif",
+        "metadata_table": "mrf_mlf",
+        "num_levels": 2
+      },
+      "executable": "tasks/create_ome_zarr.py",
       "input_type": "image",
       "meta": {
         "cpus_per_task": 1,
@@ -128,7 +134,7 @@
         "title": "YokogawaToOmeZarr",
         "type": "object"
       },
-      "executable": "yokogawa_to_ome_zarr.py",
+      "executable": "tasks/yokogawa_to_ome_zarr.py",
       "input_type": "zarr",
       "meta": {
         "cpus_per_task": 1,
@@ -189,7 +195,11 @@
         "title": "CopyOmeZarr",
         "type": "object"
       },
-      "executable": "copy_ome_zarr.py",
+      "default_args": {
+        "project_to_2D": true,
+        "suffix": "mip"
+      },
+      "executable": "tasks/copy_ome_zarr.py",
       "input_type": "zarr",
       "meta": {
         "cpus_per_task": 1,
@@ -235,7 +245,7 @@
         "title": "MaximumIntensityProjection",
         "type": "object"
       },
-      "executable": "maximum_intensity_projection.py",
+      "executable": "tasks/maximum_intensity_projection.py",
       "input_type": "zarr",
       "meta": {
         "cpus_per_task": 1,
@@ -394,7 +404,7 @@
         "title": "CellposeSegmentation",
         "type": "object"
       },
-      "executable": "cellpose_segmentation.py",
+      "executable": "tasks/cellpose_segmentation.py",
       "input_type": "zarr",
       "meta": {
         "cpus_per_task": 4,
@@ -465,7 +475,11 @@
         "title": "IlluminationCorrection",
         "type": "object"
       },
-      "executable": "illumination_correction.py",
+      "default_args": {
+        "background": 100,
+        "overwrite": false
+      },
+      "executable": "tasks/illumination_correction.py",
       "input_type": "zarr",
       "meta": {
         "cpus_per_task": 1,
@@ -566,7 +580,13 @@
         "title": "NapariWorkflowsWrapper",
         "type": "object"
       },
-      "executable": "napari_workflows_wrapper.py",
+      "default_args": {
+        "expected_dimensions": 3,
+        "input_ROI_table": "FOV_ROI_table",
+        "level": 0,
+        "relabeling": true
+      },
+      "executable": "tasks/napari_workflows_wrapper.py",
       "input_type": "zarr",
       "meta": {
         "cpus_per_task": 8,
@@ -664,7 +684,13 @@
         "title": "CreateOmeZarrMultiplex",
         "type": "object"
       },
-      "executable": "create_ome_zarr_multiplex.py",
+      "default_args": {
+        "coarsening_xy": 2,
+        "image_extension": "tif",
+        "metadata_table": "mrf_mlf",
+        "num_levels": 2
+      },
+      "executable": "tasks/create_ome_zarr_multiplex.py",
       "input_type": "image",
       "meta": {
         "cpus_per_task": 1,

fractal_tasks_core/dev/lib_args_schemas.py

@@ -83,11 +83,12 @@ def _get_args_descriptions(executable) -> dict[str, str]:
     """
     # Read docstring (via ast)
     module_path = Path(fractal_tasks_core.__file__).parent / executable
+    module_name = module_path.with_suffix("").name
     tree = ast.parse(module_path.read_text())
     function = next(
         f
         for f in ast.walk(tree)
-        if (isinstance(f, ast.FunctionDef) and f.name == executable[:-3])
+        if (isinstance(f, ast.FunctionDef) and f.name == module_name)
     )
     docstring = ast.get_docstring(function)
     # Parse docstring (via docstring_parser) and prepare output
@@ -126,8 +127,8 @@ def create_schema_for_single_task(executable: str) -> _Schema:
     if not executable.endswith(".py"):
         raise ValueError(f"Invalid {executable=} (it must end with `.py`).")
     # Import function
-    module_name = executable[:-3]
-    module = import_module(f"fractal_tasks_core.{module_name}")
+    module_name = Path(executable).with_suffix("").name
+    module = import_module(f"fractal_tasks_core.tasks.{module_name}")
     task_function = getattr(module, module_name)
 
     # Create and clean up schema

fractal_tasks_core/dev/create_args_schemas.py → fractal_tasks_core/dev/new_args_schema.py

@@ -12,7 +12,7 @@
     University of Zurich.
 
 
-Script to generate JSON schemas for task arguments afresh, and writes them
+Script to generate JSON schemas for task arguments afresh, and write them
 to the package manifest.
 """
 import json