BLD/CI: re-enable ILP64 usage and PyPy job in Azure

Also improve the `show_config` output (fixes one of the issues
discussed in numpygh-24200), and improve handling of BLAS related logic
in `numpy/meson.build`.

The `OPENBLAS64_` environment variable is only used in
`numpy.distutils.system_info`, so remove it from the places where we
no longer use `numpy.distutils`.

azure-pipelines.yml

@@ -220,15 +220,13 @@ stages:
             PYTHON_ARCH: 'x64'
             TEST_MODE: full
             BITS: 64
-            # NPY_USE_BLAS_ILP64: '1'
-            # TODO: failing because of an issue with ILP64 BLAS being downloaded,
-            #       and that not being handled automatically yet
-            # PyPy39-64bit-fast:
-              # PYTHON_VERSION: 'pypy3.9'
-              # PYTHON_ARCH: 'x64'
-              # TEST_MODE: fast
-              # BITS: 64
-              # NPY_USE_BLAS_ILP64: '1'
+            NPY_USE_BLAS_ILP64: '1'
+          PyPy39-64bit-fast:
+            PYTHON_VERSION: 'pypy3.9'
+            PYTHON_ARCH: 'x64'
+            TEST_MODE: fast
+            BITS: 64
+            NPY_USE_BLAS_ILP64: '1'
 
     steps:
     - template: azure-steps-windows.yml

azure-steps-windows.yml

@@ -36,17 +36,14 @@ steps:
   displayName: 'Download / Install OpenBLAS'
 
 - powershell: |
+    python -c "from tools import openblas_support; openblas_support.make_init('numpy')"
     If ( Test-Path env:NPY_USE_BLAS_ILP64 ) {
-        $env:OPENBLAS64_="openblas"
-        # This does not work, the flags are passed together and not separately
-        $env:CFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64"
-        $env:CXXFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64"
+        python -m pip install . -Csetup-args="--vsenv" -Csetup-args="-Duse-ilp64=true" -Csetup-args="-Dblas-symbol-suffix=64_"
     } else {
-        $env:OPENBLAS = "openblas"
+        python -m pip install . -Csetup-args="--vsenv"
     }
-    python -c "from tools import openblas_support; openblas_support.make_init('numpy')"
-    python -m pip install . --config-settings=setup-args="--vsenv"
-    # copy from c:/opt/openblas/openblas_dll to numpy/.libs
+    # copy from c:/opt/openblas/openblas_dll to numpy/.libs to ensure it can
+    # get loaded when numpy is imported (no RPATH on Windows)
     $target = $(python -c "import sysconfig; print(sysconfig.get_path('platlib'))")
     mkdir $target/numpy/.libs
     copy C:/opt/openblas/openblas_dll/*.dll $target/numpy/.libs

numpy/meson.build

@@ -50,11 +50,13 @@ endif
 
 
 # This is currently injected directly into CFLAGS/CXXFLAGS for wheel builds
-# (see cibuildwheel settings in pyproject.toml)
+# (see cibuildwheel settings in pyproject.toml), but used by CI jobs already
 blas_symbol_suffix = get_option('blas-symbol-suffix')
 
 
-# TODO: 64-bit BLAS and LAPACK
+# TODO: 64-bit (ILP64) BLAS and LAPACK support (e.g., check for more .pc files
+# so we detect `openblas64_.so` directly). Partially supported now, needs more
+# auto-detection.
 #
 # Note that this works as long as BLAS and LAPACK are detected properly via
 # pkg-config. By default we look for OpenBLAS, other libraries can be configured via
@@ -119,17 +121,6 @@ if have_blas
   endif
 endif
 
-if lapack_name == 'openblas'
-  lapack_name = ['openblas', 'OpenBLAS']
-endif
-lapack_dep = dependency(lapack_name, required: false)
-have_lapack = lapack_dep.found()
-
-dependency_map = {
-  'BLAS': blas,
-  'LAPACK': lapack_dep,
-}
-
 use_ilp64 = get_option('use-ilp64')
 if not use_ilp64
   # For now, keep supporting this environment variable too (same as in setup.py)
@@ -144,27 +135,35 @@ if not use_ilp64
 endif
 
 # BLAS and LAPACK are optional dependencies for NumPy. We can only use a BLAS
-# which provides a CBLAS interface.
-# TODO: add ILP64 support
+# which provides a CBLAS interface. So disable BLAS completely if CBLAS is not
+# found (lapack-lite will be used instead; xref gh-24200 for a discussion on
+# whether this silent disabling should stay as-is)
 if have_blas
-  c_args_blas = []  # note: used for C and C++ via `blas_dep` below
+  _args_blas = []  # note: used for C and C++ via `blas_dep` below
   if have_cblas
-    c_args_blas += ['-DHAVE_CBLAS']
+    _args_blas += ['-DHAVE_CBLAS']
   endif
   if use_ilp64
-    c_args_blas += ['-DHAVE_BLAS_ILP64']
+    _args_blas += ['-DHAVE_BLAS_ILP64']
   endif
   if blas_symbol_suffix != ''
-    c_args_blas += ['-DBLAS_SYMBOL_SUFFIX=' + blas_symbol_suffix]
+    _args_blas += ['-DBLAS_SYMBOL_SUFFIX=' + blas_symbol_suffix]
   endif
   blas_dep = declare_dependency(
     dependencies: [blas, cblas],
-    compile_args: c_args_blas,
+    compile_args: _args_blas,
   )
 else
   blas_dep = []
 endif
 
+if lapack_name == 'openblas'
+  lapack_name = ['openblas', 'OpenBLAS']
+endif
+lapack_dep = dependency(lapack_name, required: false)
+have_lapack = lapack_dep.found()
+
+
 # Copy the main __init__.py|pxd files to the build dir (needed for Cython)
 __init__py = fs.copyfile('__init__.py')
 __init__pxd = fs.copyfile('__init__.pxd')
@@ -275,7 +274,20 @@ conf_data.set('CROSS_COMPILED', meson.is_cross_build())
 conf_data.set('PYTHON_PATH', py.full_path())
 conf_data.set('PYTHON_VERSION', py.language_version())
 
-# Dependencies information
+# BLAS/LAPACK dependency info. Ensure we report dependencies correctly for
+# `np.show_config()`; needs some special handling for the case BLAS was found
+# but CBLAS not (and hence BLAS was also disabled)
+dependency_map = {
+  'LAPACK': lapack_dep,
+}
+if have_blas and have_cblas
+  dependency_map += {'BLAS': blas}
+else
+  conf_data.set('BLAS_NAME', blas_name)
+  conf_data.set('BLAS_FOUND', false)
+endif
+
+
 foreach name, dep : dependency_map
   conf_data.set(name + '_NAME', dep.name())
   conf_data.set(name + '_FOUND', dep.found())

pyproject.toml

@@ -154,7 +154,7 @@ test-command = "bash {project}/tools/wheels/cibw_test_command.sh {project}"
 manylinux-x86_64-image = "manylinux2014"
 manylinux-aarch64-image = "manylinux2014"
 musllinux-x86_64-image = "musllinux_1_1"
-environment = { CFLAGS="-fno-strict-aliasing -DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", LDFLAGS="-Wl,--strip-debug", CXXFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", OPENBLAS64_="/usr/local", NPY_USE_BLAS_ILP64="1", RUNNER_OS="Linux"}
+environment = { CFLAGS="-fno-strict-aliasing -DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", LDFLAGS="-Wl,--strip-debug", CXXFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", NPY_USE_BLAS_ILP64="1", RUNNER_OS="Linux"}
 
 [tool.cibuildwheel.macos]
 # For universal2 wheels, we will need to fuse them manually
@@ -165,19 +165,20 @@ environment = { CFLAGS="-fno-strict-aliasing -DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLA
 archs = "x86_64 arm64"
 test-skip = "*_universal2:arm64"
 # MACOS linker doesn't support stripping symbols
-environment = { CFLAGS="-fno-strict-aliasing -DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", CXXFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", OPENBLAS64_="/usr/local", NPY_USE_BLAS_ILP64="1", CC="clang", CXX = "clang++", RUNNER_OS="macOS"}
+environment = {CFLAGS="-fno-strict-aliasing -DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", CXXFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", NPY_USE_BLAS_ILP64="1", CC="clang", CXX = "clang++", RUNNER_OS="macOS"}
 
 [tool.cibuildwheel.windows]
 archs = ['AMD64']
-environment = { OPENBLAS64_="openblas", OPENBLAS="", NPY_USE_BLAS_ILP64="1", CFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", CXXFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", LDFLAGS="", PKG_CONFIG_PATH="C:/opt/64/lib/pkgconfig"}
+environment = {NPY_USE_BLAS_ILP64="1", CFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", CXXFLAGS="-DBLAS_SYMBOL_SUFFIX=64_ -DHAVE_BLAS_ILP64", LDFLAGS="", PKG_CONFIG_PATH="C:/opt/64/lib/pkgconfig"}
 config-settings = "setup-args=--vsenv"
 repair-wheel-command = "bash ./tools/wheels/repair_windows.sh {wheel} {dest_dir}"
 
 #[[tool.cibuildwheel.overrides]]
 # Note: 32-bit Python wheel builds are skipped right now; probably needs
 # --native-file to build due to `arch != pyarch` check in Meson's `python` dependency
+# Note: uses 32-bit rather than 64-bit OpenBLAS
 #select = "*-win32"
-#environment = { OPENBLAS64_="", OPENBLAS="openblas", NPY_USE_BLAS_ILP64="0", CFLAGS="-m32", LDFLAGS="-m32", PKG_CONFIG_PATH="/opt/32/lib/pkgconfig"}
+#environment = CFLAGS="-m32", LDFLAGS="-m32", PKG_CONFIG_PATH="/opt/32/lib/pkgconfig"}
 
 [tool.spin]
 package = 'numpy'