FIX, ENH Possible fix for segfault and oversubscription of RAM in oblique splitters

.circleci/config.yml

@@ -81,12 +81,14 @@ jobs:
       - run:
           name: setup Python venv
           command: |
-            pip install -r build_requirements.txt
-            pip install -r doc_requirements.txt
+            pip install --upgrade pip spin
+            spin setup-submodule
+            pip install .[build]
+            pip install .[doc]
       - run:
           name: build scikit-tree
           command: |
-            python spin build -j2
+            spin build -j2
 
       - save_cache:
           key: deps_ccache-{{ .Branch }}
@@ -127,7 +129,7 @@ jobs:
             echo "After..."
             ls build-install/**
             export PYTHONPATH=$PWD/build-install/usr/lib/python3.8/site-packages
-            python ./spin docs
+            ./spin docs
 
       - store_artifacts:
           path: doc/_build/html

.github/workflows/build_wheels.yml

@@ -60,9 +60,10 @@ jobs:
 
       - name: Setup submodule
         run: |
-          python -m pip install -r build_requirements.txt
+          python -m pip install --upgrade pip
           python -m pip install spin
-          python spin setup-submodule
+          python -m spin setup-submodule
+          python -m pip install .[build]
 
       - name: win_amd64 - install rtools
         run: |
@@ -79,7 +80,7 @@ jobs:
         # configuration step to create a build directory. The subsequent wheel
         # build then needs to use that directory. This can be done with pip
         # using a command like:
-        # python -m pip wheel --config-settings builddir=build .
+        # pip wheel --config-settings builddir=build .
         # if: >-
         #   ( ! contains(matrix.os[2], 'arm64' ) )
         env:
@@ -114,10 +115,10 @@ jobs:
 
       - name: Install cibuildwheel dependencies
         run: |
-          python -m pip install cibuildwheel
-          python -m pip install -r build_requirements.txt
-          python -m pip install spin
-          python spin setup-submodule
+          pip install cibuildwheel
+          pip install spin
+          spin setup-submodule
+          pip install .[build]
 
       - name: Build wheels
         uses: pypa/cibuildwheel@v2.16.5
@@ -146,9 +147,9 @@ jobs:
 
       - name: Build source distribution
         run: |
-          python -m pip install -r build_requirements.txt
-          python -m pip install spin
-          python spin setup-submodule
+          pip install spin build
+          spin setup-submodule
+          pip install .[build]
           python -m build
 
       - name: Store artifacts

.github/workflows/main.yml

@@ -46,28 +46,32 @@ jobs:
           python-version: ${{ matrix.python-version }}
           architecture: "x64"
           cache: "pip"
-          cache-dependency-path: "requirements.txt"
+          cache-dependency-path: "pyproject.toml"
 
-      - name: show-gcc
+      - name: Install packages for Ubuntu
+        if: ${{ matrix.os == 'ubuntu-22.04'}}
         run: |
-          gcc --version
+          sudo apt-get update
+          sudo apt-get install -y libopenblas-dev libatlas-base-dev liblapack-dev gfortran libgmp-dev libmpfr-dev libsuitesparse-dev ccache libmpc-dev
+          sudo apt-get install -y gcc
+          sudo apt-get update
 
       - name: Install Ccache for MacOSX
         if: ${{ matrix.os == 'macos-latest'}}
         run: |
           brew install ccache
+          brew install gcc
 
-      - name: Install packages for Ubuntu
-        if: ${{ matrix.os == 'ubuntu-22.04'}}
+      - name: show-gcc
         run: |
-          sudo apt-get update
-          sudo apt-get install -y libopenblas-dev libatlas-base-dev liblapack-dev gfortran libgmp-dev libmpfr-dev libsuitesparse-dev ccache libmpc-dev
+          gcc --version
 
       - name: Install Python packages
         run: |
-          python -m pip install -r build_requirements.txt
           python -m pip install spin
-          python -m pip install -r test_requirements.txt
+          python -m spin setup-submodule
+          python -m pip install .[build]
+          python -m pip install .[test]
 
       - name: Prepare compiler cache
         id: prep-ccache
@@ -153,7 +157,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           architecture: "x64"
           cache: "pip"
-          cache-dependency-path: "requirements.txt"
+          cache-dependency-path: "pyproject.toml"
 
       - name: show-gcc
         run: |
@@ -172,9 +176,10 @@ jobs:
 
       - name: Install Python packages
         run: |
-          python -m pip install -r build_requirements.txt
           python -m pip install spin
-          python -m pip install -r test_requirements.txt
+          python -m spin setup-submodule
+          python -m pip install .[build]
+          python -m pip install .[test]
 
       - name: Prepare compiler cache
         id: prep-ccache
@@ -232,7 +237,7 @@ jobs:
       - name: Upload coverage stats to codecov
         uses: codecov/codecov-action@v4
         with:
-          # python spin goes into the INSTALLED path in order to run pytest
+          # spin goes into the INSTALLED path in order to run pytest
           files: ./coverage.xml
           fail_ci_if_error: true
           verbose: true
@@ -258,7 +263,7 @@ jobs:
 #         python-version: "3.10"
 #         architecture: "x64"
 #         cache: "pip"
-#         cache-dependency-path: "requirements.txt"
+#         cache-dependency-path: "pyproject.toml"
 
 #     - name: install-rtools
 #       run: |
@@ -287,8 +292,8 @@ jobs:
 
 #     - name: pip-packages
 #       run: |
-#         pip install -r build_requirements.txt
-#         pip install -r test_requirements.txt
+#         pip install .[build]
+#         pip install .[test]
 #         pip install spin
 #         pip install numpy==1.22.4
 
@@ -319,8 +324,8 @@ jobs:
 #       run: |
 #         echo "SCIPY_USE_PROPACK=1" >> $env:GITHUB_ENV
 #         echo "FORCE_SUBMODULE=True" >> $env:GITHUB_ENV
-#         python spin setup_submodule --forcesubmodule
-#         python spin build -j 2
+#         spin setup_submodule --forcesubmodule
+#         spin build -j 2
 #         # Necessary because GitHub Actions checks out the repo to D:\ while OpenBLAS
 #         # got installed to C:\ higher up. The copying with `--win-cp-openblas` fails
 #         # when things are split over drives.
@@ -329,4 +334,4 @@ jobs:
 
 #     - name: test
 #       run: |
-#         python spin test
+#         spin test

.github/workflows/release.yml

@@ -32,8 +32,8 @@ jobs:
           architecture: "x64"
       - name: Install dependencies
         run: |
-          python -m pip install --progress-bar off --upgrade pip
-          python -m pip install --progress-bar off build twine
+          pip install --progress-bar off --upgrade pip
+          pip install --progress-bar off build twine
       - name: Prepare environment
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV

.github/workflows/style.yml

@@ -41,7 +41,10 @@ jobs:
           sudo apt-get install -y libopenblas-dev libatlas-base-dev liblapack-dev
 
       - name: Install dependencies
-        run: pip install -r style_requirements.txt
+        run: |
+          pip install --upgrade pip spin
+          spin setup-submodule
+          pip install .[style]
 
       # check formatting of the code style
       - name: Check code formatting

CONTRIBUTING.md

@@ -93,7 +93,7 @@ When you're ready to contribute code to address an open issue, please follow the
 
     Once your virtual environment is activated, you can install your local clone in "editable mode" with
 
-        pip install -r build_requirements.txt
+        pip install .[build]
         pip install -e .
 
     The "editable mode" comes from the `-e` argument to `pip`, and essential just creates a symbolic link from the site-packages directory of your virtual environment to the source code in your local clone. That way any changes you make will be immediately reflected in your virtual environment.

DEVELOPING.md

@@ -39,14 +39,14 @@ Assuming these steps have worked properly and you have read and followed any nec
 
 If you are developing locally, you will need the build dependencies to compile the Cython / C++ code:
 
-    pip install -r build_requirements.txt
+    pip install .[build]
 
 Other requirements can be installed as such:
 
-    pip install -r requirements.txt
-    pip install -r style_requirements.txt
-    pip install -r test_requirements.txt
-    pip install -r doc_requirements.txt
+    pip install .
+    pip install .[style]
+    pip install .[test]
+    pip install .[doc]
 
 # Building the project from source
 

README.md

@@ -52,7 +52,7 @@ Building locally with Meson (For developers)
 Make sure you have the necessary packages installed
 
     # install build dependencies
-    pip install -r build_requirements.txt
+    pip install .[build]
 
     # you may need these optional dependencies to build scikit-learn locally
     conda install -c conda-forge joblib threadpoolctl pytest compilers llvm-openmp

doc/install.rst

@@ -41,7 +41,7 @@ Then run installation of build packages
 
 .. code-block:: bash
 
-    pip install -r build_requirements.txt
+    pip install .[build]
     pip install spin
 
     # use spin CLI to run Meson build locally

pyproject.toml

@@ -1,16 +1,16 @@
 [build-system]
 build-backend = "mesonpy"
 requires = [
-  "meson-python>=0.13.0",
+  "meson-python>=0.15.0",
   'ninja',
   # `wheel` is needed for non-isolated builds, given that `meson-python`
   # doesn't list it as a runtime requirement (at least in 0.10.0)
   # See https://github.com/FFY00/meson-python/blob/main/pyproject.toml#L4
   "wheel",
   "setuptools<=65.5",
   "packaging",
-  "Cython>=0.29.24",
-  "scikit-learn>=1.4",
+  "Cython>=3.0.8",
+  "scikit-learn>=1.4.1",
   "scipy>=1.5.0",
   "numpy>=1.25; python_version>='3.9'"
 ]
@@ -49,9 +49,9 @@ include = [
   {path = "meson.build", format = "sdist"}
 ]
 dependencies = [
-  'numpy',
+  'numpy>=1.25.0',
   'scipy>=1.5.0',
-  'scikit-learn>=1.3.1'
+  'scikit-learn>=1.4.1'
 ]
 
 [project.optional-dependencies]
@@ -68,10 +68,10 @@ build = [
   'meson-python',
   'spin',
   'doit',
-  'scikit-learn>=1.3.1',
-  'Cython>=0.29.36',
+  'scikit-learn>=1.4.1',
+  'Cython>=3.0.8',
   'ninja',
-  'numpy',
+  'numpy>=1.25.0',
   'rich-click',
   'pydevtool'
 ]

sktree/tree/_oblique_splitter.pxd

@@ -31,8 +31,12 @@ cdef struct ObliqueSplitRecord:
     float64_t impurity_left        # Impurity of the left split.
     float64_t impurity_right       # Impurity of the right split.
 
-    vector[float32_t]* proj_vec_weights  # weights of the vector (max_features,)
-    vector[intp_t]* proj_vec_indices     # indices of the features (max_features,)
+    # XXX: By storing the values, they will have to be copied into the split record
+    #     which may be inefficient.
+    # Inside the tree, they will also again be copied into a Node, which will
+    # in total incur two copies, whereas we ideally only need one copy.
+    vector[float32_t] proj_vec_weights  # weights of the vector (max_features,)
+    vector[intp_t] proj_vec_indices     # indices of the features (max_features,)
 
 
 cdef class BaseObliqueSplitter(Splitter):
@@ -43,7 +47,7 @@ cdef class BaseObliqueSplitter(Splitter):
     cdef vector[vector[float32_t]] proj_mat_weights       # nonzero weights of sparse proj_mat matrix
     cdef vector[vector[intp_t]] proj_mat_indices        # nonzero indices of sparse proj_mat matrix
 
-    # TODO: assumes all oblique splitters only work with dense data
+    # TODO: assumes all oblique splitters only work with dense data; make work with sparse arrays too
     cdef const float32_t[:, :] X
 
     # feature weights across (n_dims,)
@@ -72,8 +76,8 @@ cdef class BaseObliqueSplitter(Splitter):
         intp_t end,
         const intp_t[:] samples,
         float32_t[:] feature_values,
-        vector[float32_t]* proj_vec_weights,  # weights of the vector (max_features,)
-        vector[intp_t]* proj_vec_indices    # indices of the features (max_features,)
+        vector[float32_t]& proj_vec_weights,  # weights of the vector (max_features,)
+        vector[intp_t]& proj_vec_indices    # indices of the features (max_features,)
     ) noexcept nogil
 
     cdef int node_split(