Merge pull request #268 from stardist/dev

Changes for 0.9.0 release
stardist · Apr 23, 2024 · e853207 · e853207
2 parents 9411978 + 73f67aa
commit e853207
Show file tree

Hide file tree

Showing 25 changed files with 485 additions and 196 deletions.
diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml
@@ -17,20 +17,23 @@ jobs:
   build_wheels:
     name: Build ${{ matrix.py }} wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    env:
-        SETUPTOOLS_USE_DISTUTILS: stdlib
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-11]
-        py: [cp36, cp37, cp38, cp39, cp310, cp311]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        py: [cp36, cp37, cp38, cp39, cp310, cp311, cp312]
+        include:
+          - cibw_test_requires: 'pytest tensorflow'
+          - os: ubuntu-latest
+            py: cp37  
+            cibw_test_requires: 'pytest tensorflow urllib3<2' # ImportError: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'OpenSSL 1.0.2k-fips  26 Jan 2017'. See: https://github.com/urllib3/urllib3/issues/2168
 
     steps:
-      - uses: actions/checkout@v3
-        name: Checkout repository
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
-        name: Install Python
+      - name: Install Python
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 
@@ -61,7 +64,7 @@ jobs:
           CIBW_BUILD: "${{ matrix.py }}-*"
           CIBW_ARCHS_MACOS: x86_64 arm64
           CIBW_BUILD_VERBOSITY: 1
-          CIBW_TEST_REQUIRES: pytest tensorflow
+          CIBW_TEST_REQUIRES: "${{ matrix.cibw_test_requires }}"
           CIBW_TEST_COMMAND: pytest -v -m "not gpu" {project}
           CIBW_TEST_SKIP: "*cp36*-macosx*" # Python crashes ¯\_(ツ)_/¯
 
@@ -76,12 +79,12 @@ jobs:
           # numba is a requirement and only has wheels for manylinux2014
           CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
           CIBW_BUILD_VERBOSITY: 1
-          CIBW_TEST_REQUIRES: pytest tensorflow
+          CIBW_TEST_REQUIRES: "${{ matrix.cibw_test_requires }}"
           CIBW_TEST_COMMAND: pytest -v -m "not gpu" {project}
           # CIBW_TEST_SKIP: "*musllinux*"
 
-      - uses: actions/upload-artifact@v3
-        name: Upload wheels
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
         with:
           name: dist
           path: ./dist/*.whl
@@ -91,14 +94,14 @@ jobs:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        name: Checkout repository
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
       - name: Build sdist
         run: pipx run build --sdist
 
-      - uses: actions/upload-artifact@v3
-        name: Upload sdist
+      - name: Upload sdist
+        uses: actions/upload-artifact@v3
         with:
           name: dist
           path: dist/*.tar.gz
@@ -110,8 +113,8 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && github.event.action == 'published'
     steps:
-      - uses: actions/download-artifact@v3
-        name: Download wheels and sdist
+      - name: Download wheels and sdist
+        uses: actions/download-artifact@v3
         with:
           name: dist
           path: dist
@@ -122,8 +125,8 @@ jobs:
           ls -l dist
           test -f "dist/stardist-${{ github.ref_name }}.tar.gz"
 
-      - uses: pypa/gh-action-pypi-publish@release/v1
-        name: Publish to PyPI
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -13,16 +13,35 @@ jobs:
   test:
     name: Test (${{ matrix.os }}, py${{ matrix.python-version }}, tf${{ matrix.tensorflow }})
     runs-on: ${{ matrix.os }}
-    env:
-        SETUPTOOLS_USE_DISTUTILS: stdlib
     strategy:
       fail-fast: false
       matrix:
+        # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
         os: [ubuntu-20.04, windows-latest, macos-latest]
-        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
         tensorflow: [1, 2]
+        include:
+          # always set 'package-extras' conditional on tensorflow version
+          - tensorflow: 1
+            package-extras: 'test,tf1'
+          - tensorflow: 2
+            package-extras: 'test'
+          # test bioimage.io export with python 3.7 and tensorflow 1
+          - python-version: '3.7'
+            tensorflow: 1
+            package-extras: 'test,tf1,bioimageio'
+          # install edt with numpy>=1.20 when testing with python 3.9
+          - python-version: '3.9'
+            deps-extra: 'edt "numpy>=1.20"'
+          # add extra run to test latest tensorflow with legacy keras
+          - os: ubuntu-20.04
+            python-version: '>=3.12 <3.13'  # intentionally be different than '3.12' to cause an additional combination
+            tensorflow: 2
+            package-extras: 'test'
+            deps-extra: 'tf_keras'
+            exec-extra: 'echo "TF_USE_LEGACY_KERAS=1" >> $GITHUB_ENV'
         exclude:
-            # https://www.tensorflow.org/install/source#tested_build_configurations
+          # https://www.tensorflow.org/install/source#tested_build_configurations
           - python-version: '3.8'
             tensorflow: 1
           - python-version: '3.9'
@@ -31,27 +50,25 @@ jobs:
             tensorflow: 1
           - python-version: '3.11'
             tensorflow: 1
+          - python-version: '3.12'
+            tensorflow: 1
           - os: windows-latest
             tensorflow: 1
 
     steps:
-      - uses: actions/setup-python@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - run: python -m pip install --upgrade pip wheel setuptools
-      - run: pip install "tensorflow==${{ matrix.tensorflow }}.*"
-      - uses: actions/checkout@v3
+        
+      - name: Upgrade pip, wheel, setuptools
+        run: python -m pip install --upgrade pip wheel setuptools
 
-      - if: matrix.tensorflow == 1
-        run: echo "package_extras=test,tf1" >> $GITHUB_ENV
-        shell: bash
-      - if: matrix.tensorflow == 2
-        run: echo "package_extras=test" >> $GITHUB_ENV
-        shell: bash
+      - name: Install TensorFlow
+        run: pip install "tensorflow==${{ matrix.tensorflow }}.*"
 
-      - name: Install edt
-        if: matrix.python-version == 3.9
-        run: pip install edt "numpy>=1.20"
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
       # # should also work with clang, but there are 
       # # openmp-related symbol issues just when using Python 3.6?
@@ -63,18 +80,22 @@ jobs:
       #     echo "CC=/usr/local/bin/gcc-9" >> $GITHUB_ENV
       #     echo "CXX=/usr/local/bin/g++-9" >> $GITHUB_ENV
 
+      - name: Install extra dependencies (if any)
+        if: ${{ matrix.deps-extra }}
+        run: pip install ${{ matrix.deps-extra }}
+
       - name: Install package
-        run: pip install ".[${{env.package_extras}}]"
+        run: pip install ".[${{ matrix.package-extras }}]"
 
-      - name: Install bioimage.io dependencies
-        if: |
-          matrix.python-version == 3.7 &&
-          matrix.tensorflow == 1
-        run: pip install ".[bioimageio]"
+      - name: Run extra commands (if any)
+        if: ${{ matrix.exec-extra }}
+        run: ${{ matrix.exec-extra }}
+        shell: bash
 
       - name: Disable PytestReturnNotNoneWarning (Python 3.7+)
         if: matrix.python-version != 3.6
         run: echo "PYTEST_ADDOPTS='-W ignore::pytest.PytestReturnNotNoneWarning'" >> $GITHUB_ENV
         shell: bash
 
-      - run: pytest -m "not gpu"
+      - name: Run tests
+        run: pytest -m "not gpu"
diff --git a/.github/workflows/tests_cron.yml b/.github/workflows/tests_cron.yml
@@ -8,16 +8,35 @@ jobs:
   test:
     name: Test PyPI (${{ matrix.os }}, py${{ matrix.python-version }}, tf${{ matrix.tensorflow }})
     runs-on: ${{ matrix.os }}
-    env:
-        SETUPTOOLS_USE_DISTUTILS: stdlib
     strategy:
       fail-fast: false
       matrix:
+        # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
         os: [ubuntu-20.04, windows-latest, macos-latest]
-        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
         tensorflow: [1, 2]
+        include:
+          # always set 'package-extras' conditional on tensorflow version
+          - tensorflow: 1
+            package-extras: 'test,tf1'
+          - tensorflow: 2
+            package-extras: 'test'
+          # test bioimage.io export with python 3.7 and tensorflow 1
+          - python-version: '3.7'
+            tensorflow: 1
+            package-extras: 'test,tf1,bioimageio'
+          # install edt with numpy>=1.20 when testing with python 3.9
+          - python-version: '3.9'
+            deps-extra: 'edt "numpy>=1.20"'
+          # add extra run to test latest tensorflow with legacy keras
+          - os: ubuntu-20.04
+            python-version: '>=3.12 <3.13'  # intentionally be different than '3.12' to cause an additional combination
+            tensorflow: 2
+            package-extras: 'test'
+            deps-extra: 'tf_keras'
+            exec-extra: 'echo "TF_USE_LEGACY_KERAS=1" >> $GITHUB_ENV'
         exclude:
-            # https://www.tensorflow.org/install/source#tested_build_configurations
+          # https://www.tensorflow.org/install/source#tested_build_configurations
           - python-version: '3.8'
             tensorflow: 1
           - python-version: '3.9'
@@ -26,35 +45,37 @@ jobs:
             tensorflow: 1
           - python-version: '3.11'
             tensorflow: 1
+          - python-version: '3.12'
+            tensorflow: 1
           - os: windows-latest
             tensorflow: 1
 
     steps:
-      - uses: actions/setup-python@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - run: python -m pip install --upgrade pip wheel setuptools
-      - run: pip install "tensorflow==${{ matrix.tensorflow }}.*"
-      - uses: actions/checkout@v3
+        
+      - name: Upgrade pip, wheel, setuptools
+        run: python -m pip install --upgrade pip wheel setuptools
 
-      - if: matrix.tensorflow == 1
-        run: echo "package_extras=test,tf1" >> $GITHUB_ENV
-        shell: bash
-      - if: matrix.tensorflow == 2
-        run: echo "package_extras=test" >> $GITHUB_ENV
-        shell: bash
+      - name: Install TensorFlow
+        run: pip install "tensorflow==${{ matrix.tensorflow }}.*"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: Install edt
-        if: matrix.python-version == 3.9
-        run: pip install edt "numpy>=1.20"
+      - name: Install extra dependencies (if any)
+        if: ${{ matrix.deps-extra }}
+        run: pip install ${{ matrix.deps-extra }}
 
       - name: Install package from source (just to test wheel building)
         if: startsWith(matrix.os, 'windows') == false
-        run: pip install --no-binary stardist "stardist[${{env.package_extras}}]"
+        run: pip install --no-binary stardist "stardist[${{ matrix.package-extras }}]"
 
       - name: Install package from source (just to test wheel building) - Windows
         if: startsWith(matrix.os, 'windows')
-        run: pip install --no-binary stardist "stardist[${{env.package_extras}}]"
+        run: pip install --no-binary stardist "stardist[${{ matrix.package-extras }}]"
         env:  # cf. https://github.com/pypa/pip/issues/11231
           TEMP: "/tmp"    # notes:
           TMPDIR: "/tmp"  # - /tmp should already exist on runner image
@@ -63,17 +84,17 @@ jobs:
       - name: Install package from wheel
         run: |
           pip uninstall -y stardist
-          pip install --no-cache-dir --only-binary stardist "stardist[${{env.package_extras}}]>0"
+          pip install --no-cache-dir --only-binary stardist "stardist[${{ matrix.package-extras }}]>0"
 
-      - name: Install bioimage.io dependencies
-        if: |
-          matrix.python-version == 3.7 &&
-          matrix.tensorflow == 1
-        run: pip install "stardist[bioimageio]"
+      - name: Run extra commands (if any)
+        if: ${{ matrix.exec-extra }}
+        run: ${{ matrix.exec-extra }}
+        shell: bash
 
       - name: Disable PytestReturnNotNoneWarning (Python 3.7+)
         if: matrix.python-version != 3.6
         run: echo "PYTEST_ADDOPTS='-W ignore::pytest.PytestReturnNotNoneWarning'" >> $GITHUB_ENV
         shell: bash
 
-      - run: pytest -m "not gpu"
+      - name: Run tests
+        run: pytest -m "not gpu"
diff --git a/README.md b/README.md
@@ -44,7 +44,7 @@ If you want to know more about the concepts and practical applications of StarDi
 
 ## Installation
 
-This package is compatible with Python 3.6 - 3.11.
+This package is compatible with Python 3.6 - 3.12.
 
 If you only want to use a StarDist plugin for a GUI-based software, please read [this](#plugins-for-other-software).
 

diff --git a/examples/README.md b/examples/README.md
@@ -1,6 +1,9 @@
-GitHub often has problems displaying Jupyter notebooks. Hence, we recommend to [look at our example notebooks here](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/).
+GitHub often has problems displaying Jupyter notebooks. Hence, we recommend to [look at our example notebooks here](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/) or use the links below.
 
 - [StarDist for 2D](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/2D)
 - [StarDist for 3D](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/3D)
 
-- [Other 2D examples](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/other2D) (more details about inner workings, ImageJ ROI export)
+More examples (multi-class, big data prediction, model export, etc.):
+
+- [More 2D examples](https://github.com/stardist/stardist/tree/master/examples/other2D)
+- [More 3D examples](https://github.com/stardist/stardist/tree/master/examples/other3D)
diff --git a/examples/other2D/README.md b/examples/other2D/README.md
@@ -0,0 +1,23 @@
+GitHub often has problems displaying Jupyter notebooks. Hence, we recommend to [look at our example notebooks here](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/other2D) or use the links below.
+
+### StarDist with additional object classification
+
+- [Multi-class: Data, Training, and Prediction](https://nbviewer.jupyter.org/github/stardist/stardist/blob/master/examples/other2D/multiclass.ipynb)
+
+### Prediction for very large images
+
+- [Big data prediction](https://nbviewer.jupyter.org/github/stardist/stardist/blob/master/examples/other2D/predict_big_data.ipynb)
+
+### Export StarDist model as BioImage.io model (BioImage Model Zoo)
+
+- [BioImage.io model export](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/other2D/bioimageio.ipynb)
+
+### Export detected objects as ImageJ ROIs
+
+- [ImageJ ROI export](https://nbviewer.jupyter.org/github/stardist/stardist/blob/master/examples/other2D/export_imagej_rois.ipynb)
+
+### Older example notebooks that show more about the inner workings of StarDist
+
+1. [Data](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/other2D/1_data.ipynb)
+2. [Training](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/other2D/2_training.ipynb)
+3. [Prediction](https://nbviewer.jupyter.org/github/stardist/stardist/tree/master/examples/other2D/3_prediction.ipynb)
diff --git a/examples/other2D/bioimageio.ipynb b/examples/other2D/bioimageio.ipynb
@@ -158,7 +158,7 @@
      "text": [
       "Help on function export_bioimageio in module stardist.bioimageio_utils:\n",
       "\n",
-      "export_bioimageio(model, outpath, test_input, test_input_axes=None, test_input_norm_axes='ZYX', name=None, mode='tensorflow_saved_model_bundle', min_percentile=1.0, max_percentile=99.8, overwrite_spec_kwargs=None)\n",
+      "export_bioimageio(model, outpath, test_input, test_input_axes=None, test_input_norm_axes='ZYX', name=None, mode='tensorflow_saved_model_bundle', min_percentile=1.0, max_percentile=99.8, overwrite_spec_kwargs=None, generate_default_deps=False)\n",
       "    Export stardist model into bioimage.io format, https://github.com/bioimage-io/spec-bioimage-io.\n",
       "    \n",
       "    Parameters\n",
@@ -186,6 +186,10 @@
       "        max percentile to be used for image normalization (default: 99.8)\n",
       "    overwrite_spec_kwargs: dict or None\n",
       "        spec keywords that should be overloaded (default: None)\n",
+      "    generate_default_deps: bool\n",
+      "        not required for bioimage.io, i.e. StarDist models don't need a dependencies field in rdf.yaml (default: False)\n",
+      "        if True, generate an environment.yaml file recording the python, bioimageio.core, stardist and tensorflow requirements\n",
+      "        from which a conda environment can be recreated to run this export\n",
       "\n"
      ]
     }