[BUG] Issue installing cuML in Databricks

[jwratten]

Describe the bug
I am attempting to use cuML within Databricks. I can install without an error, but when trying to run a command like "from cuml.manifold import UMAP" I run into issues.

Steps/Code to reproduce bug
I used the following code to install cuml and there was no error.
!pip install cudf-cu11 dask-cudf-cu11 --extra-index-url=https://pypi.nvidia.com/
!pip install cuml-cu11 --extra-index-url=https://pypi.nvidia.com/
!pip install cugraph-cu11 --extra-index-url=https://pypi.nvidia.com/
!pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64

However, when running "from cuml.mainfold install UMAP" I get the following error:

Expected behavior
I expected to see no errors and be able to use cuml.cluster and UMAP.

Environment details (please complete the following information):

I am using Databricks.

Here is the runtime and GPU I am using.

Additional context
Here is the full traceback.

ValueError: pyarrow.lib.IpcWriteOptions size changed, may indicate binary incompatibility. Expected 88 from C header, got 72 from PyObject
File , line 1
----> 1 from cuml.manifold import UMAP
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/init.py:17
1 #
2 # Copyright (c) 2022-2023, NVIDIA CORPORATION.
3 #
(...)
14 # limitations under the License.
15 #
---> 17 from cuml.internals.base import Base, UniversalBase
18 from cuml.internals.available_devices import is_cuda_available
20 # GPU only packages
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/init.py:18
1 #
2 # Copyright (c) 2019-2023, NVIDIA CORPORATION.
3 #
(...)
14 # limitations under the License.
15 #
17 from cuml.internals.available_devices import is_cuda_available
---> 18 from cuml.internals.base_helpers import BaseMetaClass, _tags_class_and_instance
19 from cuml.internals.api_decorators import (
20 _deprecate_pos_args,
21 api_base_fit_transform,
(...)
33 exit_internal_api,
34 )
35 from cuml.internals.api_context_managers import (
36 in_internal_api,
37 set_api_output_dtype,
38 set_api_output_type,
39 )
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/base_helpers.py:20
17 from inspect import Parameter, signature
18 import typing
---> 20 from cuml.internals.api_decorators import (
21 api_base_return_generic,
22 api_base_return_array,
23 api_base_return_sparse_array,
24 api_base_return_any,
25 api_return_any,
26 _deprecate_pos_args,
27 )
28 from cuml.internals.array import CumlArray
29 from cuml.internals.array_sparse import SparseCumlArray
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/api_decorators.py:24
21 import warnings
23 # TODO: Try to resolve circular import that makes this necessary:
---> 24 from cuml.internals import input_utils as iu
25 from cuml.internals.api_context_managers import BaseReturnAnyCM
26 from cuml.internals.api_context_managers import BaseReturnArrayCM
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/input_utils.py:19
1 #
2 # Copyright (c) 2019-2023, NVIDIA CORPORATION.
3 #
(...)
14 # limitations under the License.
15 #
17 from collections import namedtuple
---> 19 from cuml.internals.array import CumlArray
20 from cuml.internals.array_sparse import SparseCumlArray
21 from cuml.internals.global_settings import GlobalSettings
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/array.py:21
18 import operator
19 import pickle
---> 21 from cuml.internals.global_settings import GlobalSettings
22 from cuml.internals.logger import debug
23 from cuml.internals.mem_type import MemoryType, MemoryTypeError
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/global_settings.py:20
18 import threading
19 from cuml.internals.available_devices import is_cuda_available
---> 20 from cuml.internals.device_type import DeviceType
21 from cuml.internals.mem_type import MemoryType
22 from cuml.internals.safe_imports import cpu_only_import, gpu_only_import
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/device_type.py:19
1 #
2 # Copyright (c) 2022-2023, NVIDIA CORPORATION.
3 #
(...)
14 # limitations under the License.
15 #
18 from enum import Enum, auto
---> 19 from cuml.internals.mem_type import MemoryType
22 class DeviceTypeError(Exception):
23 """An exception thrown to indicate bad device type selection"""
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/mem_type.py:22
19 from cuml.internals.device_support import GPU_ENABLED
20 from cuml.internals.safe_imports import cpu_only_import, gpu_only_import
---> 22 cudf = gpu_only_import("cudf")
23 cp = gpu_only_import("cupy")
24 cpx_sparse = gpu_only_import("cupyx.scipy.sparse")
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cuml/internals/safe_imports.py:356, in gpu_only_import(module, alt)
330 """A function used to import modules required only in GPU installs
331
332 This function will attempt to import a module with the given name, but it
(...)
353 UnavailableMeta.
354 """
355 if GPU_ENABLED:
--> 356 return importlib.import_module(module)
357 else:
358 return safe_import(
359 module,
360 msg=f"{module} is not installed in non GPU-enabled installations",
361 alt=alt,
362 )
File /usr/lib/python3.10/importlib/init.py:126, in import_module(name, package)
124 break
125 level += 1
--> 126 return _bootstrap._gcd_import(name[level:], package, level)
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cudf/init.py:9
6 from cudf.utils._numba import _setup_numba
7 from cudf.utils.gpu_utils import validate_setup
----> 9 _setup_numba()
10 validate_setup()
12 import cupy
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cudf/utils/_numba.py:124, in _setup_numba()
121 if versions != NO_DRIVER:
122 driver_version, runtime_version = versions
123 ptx_toolkit_version = _get_cuda_version_from_ptx_file(
--> 124 _get_cc_60_ptx_file()
125 )
127 # MVC is required whenever any PTX is newer than the driver
128 # This could be the shipped PTX file or the PTX emitted by
129 # the version of NVVM on the user system, the latter aligning
130 # with the runtime version
131 if (driver_version < ptx_toolkit_version) or (
132 driver_version < runtime_version
133 ):
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cudf/utils/_numba.py:16, in _get_cc_60_ptx_file()
14 @lru_cache
15 def _get_cc_60_ptx_file():
---> 16 from cudf._lib import strings_udf
18 return os.path.join(
19 os.path.dirname(strings_udf.file),
20 "..",
(...)
23 "shim_60.ptx",
24 )
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cudf/_lib/init.py:4
1 # Copyright (c) 2020-2023, NVIDIA CORPORATION.
2 import numpy as np
----> 4 from . import (
5 avro,
6 binaryop,
7 concat,
8 copying,
9 csv,
10 datetime,
11 expressions,
12 filling,
13 groupby,
14 hash,
15 interop,
16 join,
17 json,
18 labeling,
19 merge,
20 null_mask,
21 nvtext,
22 orc,
23 parquet,
24 partitioning,
25 pylibcudf,
26 quantiles,
27 reduce,
28 replace,
29 reshape,
30 rolling,
31 round,
32 search,
33 sort,
34 stream_compaction,
35 string_casting,
36 strings,
37 strings_udf,
38 text,
39 timezone,
40 transpose,
41 unary,
42 )
44 MAX_COLUMN_SIZE = np.iinfo(np.int32).max
45 MAX_COLUMN_SIZE_STR = "INT32_MAX"
File avro.pyx:1, in init cudf._lib.avro()
File column.pyx:1, in init cudf._lib.column()
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-8dfbed63-3db6-4d3e-8d2e-a0c69f66111a/lib/python3.10/site-packages/cudf/_lib/pylibcudf/init.py:3
1 # Copyright (c) 2023, NVIDIA CORPORATION.
----> 3 from . import copying, interop
4 from .column import Column
5 from .gpumemoryview import gpumemoryview
File copying.pyx:1, in init cudf._lib.pylibcudf.copying()

[rayenFathallah]

I have the same problem with Kaggle notebooks, and I think it is due to library versioning. I hope we get an answer to this

[dantegd]

Apologies for my delay, this issue was opened just before the release3 of 24.04. I was wondering if this is still happening in the latest release, and if you could provide the output of https://github.com/rapidsai/cuml/blob/branch-24.06/print_env.sh?