It/keras3 pytorch

.github/workflows/test_and_deploy.yml

@@ -41,10 +41,10 @@ jobs:
       matrix:
         # Run all supported Python versions on linux
         os: [ubuntu-latest]
-        python-version: ["3.9", "3.10"]
-        # Include one macos run
+        python-version: ["3.9", "3.10", "3.11"]
+        # Include one windows, one macos run (intel based macOS 13 runner)
         include:
-        - os: macos-latest
+        - os: macos-13
           python-version: "3.10"
         - os: windows-latest
           python-version: "3.10"
@@ -54,8 +54,8 @@ jobs:
       - name: Cache Keras model
         uses: actions/cache@v3
         with:
-          path: "~/.cellfinder"
-          key: models-${{ hashFiles('~/.cellfinder/**') }}
+          path: "~/.brainglobe"
+          key: models-${{ hashFiles('~/.brainglobe/**') }}
       # Setup pyqt libraries
       - name: Setup qtpy libraries
         uses: tlambert03/setup-qt-libs@v1
@@ -65,11 +65,13 @@ jobs:
       - uses: neuroinformatics-unit/actions/test@v2
         with:
           python-version: ${{ matrix.python-version }}
+          secret-codecov-token: ${{ secrets.CODECOV_TOKEN }}
           use-xvfb: true
 
   test_numba_disabled:
     needs: [linting, manifest]
     name: Run tests with numba disabled
+    timeout-minutes: 60
     runs-on: ubuntu-latest
     env:
        NUMBA_DISABLE_JIT: "1"
@@ -78,8 +80,8 @@ jobs:
       - name: Cache Keras model
         uses: actions/cache@v3
         with:
-          path: "~/.cellfinder"
-          key: models-${{ hashFiles('~/.cellfinder/**') }}
+          path: "~/.brainglobe"
+          key: models-${{ hashFiles('~/.brainglobe/**') }}
       # Setup pyqt libraries
       - name: Setup qtpy libraries
         uses: tlambert03/setup-qt-libs@v1
@@ -89,22 +91,24 @@ jobs:
       - uses: neuroinformatics-unit/actions/test@v2
         with:
           python-version: "3.10"
+          secret-codecov-token: ${{ secrets.CODECOV_TOKEN }}
           codecov-flags: "numba"
 
   # Run brainglobe-workflows brainmapper-CLI tests to check for
   # breakages
   test_brainmapper_cli:
     needs: [linting, manifest]
     name: Run brainmapper tests to check for breakages
+    timeout-minutes: 60
     runs-on: ubuntu-latest
     env:
-      KERAS_BACKEND: jax
+      KERAS_BACKEND: torch
     steps:
       - name: Cache Keras model
         uses: actions/cache@v3
         with:
-          path: "~/.cellfinder"
-          key: models-${{ hashFiles('~/.cellfinder/**') }}
+          path: "~/.brainglobe"
+          key: models-${{ hashFiles('~/.brainglobe/**') }}
 
       - name: Checkout brainglobe-workflows
         uses: actions/checkout@v3
@@ -119,8 +123,9 @@ jobs:
       - name: Install test dependencies
         run: |
           python -m pip install --upgrade pip wheel
-          # Install cellfinder from the latest SHA on this branch (Keras with JAX backend)
-          python -m pip install "cellfinder[jax] @ git+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA"
+          # Install cellfinder from the latest SHA on this branch
+          python -m pip install "cellfinder @ git+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA"
+
           # Install checked out copy of brainglobe-workflows
           python -m pip install .[dev]
 

.github/workflows/test_include_guard.yaml

@@ -24,11 +24,13 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Install cellfinder via pip, specifying tensorflow as keras' backend
-        run: python -m pip install -e ".[tf]"
+      - name: Install cellfinder via pip
+        run: python -m pip install -e "."
 
       - name: Test (working) import
         uses: jannekem/run-python-script-action@v1
+        env:
+          KERAS_BACKEND: torch
         with:
           fail-on-error: true
           script: |

.gitignore

@@ -130,11 +130,6 @@ mprofile*.dat
 
 *.DS_Store
 
-# asv
-.asv
-benchmarks/results
-benchmarks/html
-benchmarks/env
 
 # OS
 .DS_Store

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ ci:
 
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.5.0
+      rev: v4.6.0
       hooks:
           - id: check-docstring-first
           - id: check-executables-have-shebangs
@@ -16,10 +16,10 @@ repos:
           - id: requirements-txt-fixer
           - id: trailing-whitespace
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.3.5
+      rev: v0.4.3
       hooks:
         - id: ruff
     - repo: https://github.com/psf/black
-      rev: 24.3.0
+      rev: 24.4.2
       hooks:
           - id: black

MANIFEST.in

@@ -12,7 +12,7 @@ exclude tox.ini
 graft cellfinder
 include cellfinder/napari/napari.yaml
 
-prune benchmarks
 prune examples
 prune resources
 prune tests
+prune benchmarks

benchmarks/README.md

@@ -1,39 +1,12 @@
-# Benchmarking with asv
-[Install asv](https://asv.readthedocs.io/en/stable/installing.html) by running:
-```
-pip install asv
-```
-
-`asv` works roughly as follows:
-1. It creates a virtual environment (as defined in the config)
-2. It installs the software package version of a specific commit (or of a local commit)
-3. It times the benchmarking tests and saves the results to json files
-4. The json files are 'published' into an html dir
-5. The html dir can be visualised in a static website
-
-## Running benchmarks
-To run benchmarks on a specific commit:
-```
-$ asv run 88fbbc33^!
-```
-
-To run them up to a specific commit:
-```
-$ asv run 88fbbc33
-```
-
-To run them on a range of commits:
-```
-$ asv run 827f322b..729abcf3
-```
-
-To collate the benchmarks' results into a viewable website:
-```
-$ asv publish
-```
-This will create a tree of files in the `html` directory, but this cannot be viewed directly from the local filesystem, so we need to put them in a static site. `asv publish` also detects statistically significant decreases of performance, the results can be inspected in the 'Regression' tab of the static site.
-
-To visualise the results in a static site:
-```
-$ asv preview
-```
+# Benchmarks
+`detect_and_classify.py` contains a simple script that runs
+detection and classification with the small test dataset.
+
+## Memory
+[memory_profiler](https://github.com/pythonprofilers/memory_profiler)
+can be used to profile memory useage. Install, and then run
+`mprof run --include-children --multiprocess detect_and_classify.py`. It is **very**
+important to use these two flags to capture memory usage by the additional
+processes that cellfinder.core uses.
+
+To show the results of the latest profile run, run `mprof plot`.