Failed to load cmip6 data using to_dataset_dict function

[zhentaom4w]

Hi all, Iam trying to create a simple example to load cmip6 data.
Everything is fine before dset_dict_raw = cat_pp.to_dataset_dict(xarray_open_kwargs={"consolidated": True}).

Then I got error when converting my subset to dictionary.

-> The keys in the returned dictionary of datasets are constructed as follows:
'activity_id.institution_id.source_id.experiment_id.table_id.grid_label'

0.00% [0/1 00:00<?]

ModuleNotFoundError Traceback (most recent call last)
File ~\miniconda3\lib\site-packages\fsspec\registry.py:243, in get_filesystem_class(protocol)
242 try:
--> 243 register_implementation(protocol, _import_class(bit["class"]))
244 except ImportError as e:

File ~\miniconda3\lib\site-packages\fsspec\registry.py:266, in _import_class(cls, minv)
265 mod, name = cls.rsplit(".", 1)
--> 266 mod = importlib.import_module(mod)
267 return getattr(mod, name)

File ~\miniconda3\lib\importlib_init_.py:126, in import_module(name, package)
125 level += 1
--> 126 return _bootstrap._gcd_import(name[level:], package, level)

File :1050, in _gcd_import(name, package, level)

File :1027, in find_and_load(name, import)

File :1004, in find_and_load_unlocked(name, import)

ModuleNotFoundError: No module named 'gcsfs'

The above exception was the direct cause of the following exception:

ImportError Traceback (most recent call last)
File ~\miniconda3\lib\site-packages\intake_esm\source.py:240, in ESMDataSource._open_dataset(self)
220 datasets = [
221 _open_dataset(
222 record[self.path_column_name],
(...)
237 for _, record in self.df.iterrows()
238 ]
--> 240 datasets = dask.compute(*datasets)
241 if len(datasets) == 1:

File ~\miniconda3\lib\site-packages\dask\base.py:600, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
598 postcomputes.append(x.dask_postcompute())
--> 600 results = schedule(dsk, keys, **kwargs)
601 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])

File ~\miniconda3\lib\site-packages\dask\threaded.py:89, in get(dsk, keys, cache, num_workers, pool, **kwargs)
87 pool = MultiprocessingPoolExecutor(pool)
---> 89 results = get_async(
90 pool.submit,
91 pool._max_workers,
92 dsk,
93 keys,
94 cache=cache,
95 get_id=_thread_get_id,
96 pack_exception=pack_exception,
97 **kwargs,
98 )
100 # Cleanup pools associated to dead threads

File ~\miniconda3\lib\site-packages\dask\local.py:511, in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)
510 else:
--> 511 raise_exception(exc, tb)
512 res, worker_id = loads(res_info)

File ~\miniconda3\lib\site-packages\dask\local.py:319, in reraise(exc, tb)
318 raise exc.with_traceback(tb)
--> 319 raise exc

File ~\miniconda3\lib\site-packages\dask\local.py:224, in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
223 task, data = loads(task_info)
--> 224 result = _execute_task(task, data)
225 id = get_id()

File ~\miniconda3\lib\site-packages\dask\core.py:119, in _execute_task(arg, cache, dsk)
116 # Note: Don't assign the subtask results to a variable. numpy detects
117 # temporaries by their reference count and can execute certain
118 # operations in-place.
--> 119 return func(*(_execute_task(a, cache) for a in args))
120 elif not ishashable(arg):

File ~\miniconda3\lib\site-packages\dask\utils.py:71, in apply(func, args, kwargs)
70 if kwargs:
---> 71 return func(*args, **kwargs)
72 else:

File ~\miniconda3\lib\site-packages\intake_esm\source.py:73, in _open_dataset(urlpath, varname, xarray_open_kwargs, preprocess, requested_variables, additional_attrs, expand_dims, data_format)
72 else:
---> 73 ds = xr.open_dataset(url, **xarray_open_kwargs)
74 if preprocess is not None:

File ~\miniconda3\lib\site-packages\xarray\backends\api.py:540, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, backend_kwargs, **kwargs)
539 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 540 backend_ds = backend.open_dataset(
541 filename_or_obj,
542 drop_variables=drop_variables,
543 **decoders,
544 **kwargs,
545 )
546 ds = _dataset_from_backend_dataset(
547 backend_ds,
548 filename_or_obj,
(...)
556 **kwargs,
557 )

File ~\miniconda3\lib\site-packages\xarray\backends\zarr.py:882, in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel, zarr_version)
881 filename_or_obj = _normalize_path(filename_or_obj)
--> 882 store = ZarrStore.open_group(
883 filename_or_obj,
884 group=group,
885 mode=mode,
886 synchronizer=synchronizer,
887 consolidated=consolidated,
888 consolidate_on_close=False,
889 chunk_store=chunk_store,
890 storage_options=storage_options,
891 stacklevel=stacklevel + 1,
892 zarr_version=zarr_version,
893 )
895 store_entrypoint = StoreBackendEntrypoint()

File ~\miniconda3\lib\site-packages\xarray\backends\zarr.py:418, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, stacklevel, zarr_version)
416 elif consolidated:
417 # TODO: an option to pass the metadata_key keyword
--> 418 zarr_group = zarr.open_consolidated(store, **open_kwargs)
419 else:

File ~\miniconda3\lib\site-packages\zarr\convenience.py:1283, in open_consolidated(store, metadata_key, mode, **kwargs)
1282 zarr_version = kwargs.get('zarr_version')
-> 1283 store = normalize_store_arg(store, storage_options=kwargs.get("storage_options"), mode=mode,
1284 zarr_version=zarr_version)
1285 if mode not in {'r', 'r+'}:

File ~\miniconda3\lib\site-packages\zarr\storage.py:171, in normalize_store_arg(store, storage_options, mode, zarr_version)
170 normalize_store = _normalize_store_arg_v3
--> 171 return normalize_store(store, storage_options, mode)

File ~\miniconda3\lib\site-packages\zarr\storage.py:144, in _normalize_store_arg_v2(store, storage_options, mode)
143 if "://" in store or "::" in store:
--> 144 return FSStore(store, mode=mode, **(storage_options or {}))
145 elif storage_options:

File ~\miniconda3\lib\site-packages\zarr\storage.py:1328, in FSStore.init(self, url, normalize_keys, key_separator, mode, exceptions, dimension_separator, fs, check, create, missing_exceptions, **storage_options)
1327 storage_options["auto_mkdir"] = True
-> 1328 self.map = fsspec.get_mapper(url, **{**mapper_options, **storage_options})
1329 self.fs = self.map.fs # for direct operations

File ~\miniconda3\lib\site-packages\fsspec\mapping.py:237, in get_mapper(url, check, create, missing_exceptions, alternate_root, **kwargs)
236 # Removing protocol here - could defer to each open() on the backend
--> 237 fs, urlpath = url_to_fs(url, **kwargs)
238 root = alternate_root if alternate_root is not None else urlpath

File ~\miniconda3\lib\site-packages\fsspec\core.py:382, in url_to_fs(url, **kwargs)
381 protocol = split_protocol(url)[0]
--> 382 cls = get_filesystem_class(protocol)
384 options = cls._get_kwargs_from_urls(url)

File ~\miniconda3\lib\site-packages\fsspec\registry.py:245, in get_filesystem_class(protocol)
244 except ImportError as e:
--> 245 raise ImportError(bit["err"]) from e
246 cls = registry[protocol]

ImportError: Please install gcsfs to access Google Storage

The above exception was the direct cause of the following exception:

ESMDataSourceError Traceback (most recent call last)
Cell In[34], line 1
----> 1 dset_dict_raw = cat_pp.to_dataset_dict(xarray_open_kwargs={"consolidated": True})

File ~\miniconda3\lib\site-packages\pydantic\decorator.py:40, in pydantic.decorator.validate_arguments.validate.wrapper_function()

File ~\miniconda3\lib\site-packages\pydantic\decorator.py:134, in pydantic.decorator.ValidatedFunction.call()

File ~\miniconda3\lib\site-packages\pydantic\decorator.py:206, in pydantic.decorator.ValidatedFunction.execute()

File ~\miniconda3\lib\site-packages\intake_esm\core.py:651, in esm_datastore.to_dataset_dict(self, xarray_open_kwargs, xarray_combine_by_coords_kwargs, preprocess, storage_options, progressbar, aggregate, skip_on_error, **kwargs)
649 except Exception as exc:
650 if not skip_on_error:
--> 651 raise exc
652 self.datasets = self._create_derived_variables(datasets, skip_on_error)
653 return self.datasets

File ~\miniconda3\lib\site-packages\intake_esm\core.py:647, in esm_datastore.to_dataset_dict(self, xarray_open_kwargs, xarray_combine_by_coords_kwargs, preprocess, storage_options, progressbar, aggregate, skip_on_error, **kwargs)
645 for task in gen:
646 try:
--> 647 key, ds = task.result()
648 datasets[key] = ds
649 except Exception as exc:

File ~\miniconda3\lib\concurrent\futures_base.py:451, in Future.result(self, timeout)
449 raise CancelledError()
450 elif self._state == FINISHED:
--> 451 return self.__get_result()
453 self._condition.wait(timeout)
455 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:

File ~\miniconda3\lib\concurrent\futures_base.py:403, in Future.__get_result(self)
401 if self._exception:
402 try:
--> 403 raise self._exception
404 finally:
405 # Break a reference cycle with the exception in self._exception
406 self = None

File ~\miniconda3\lib\concurrent\futures\thread.py:58, in _WorkItem.run(self)
55 return
57 try:
---> 58 result = self.fn(*self.args, **self.kwargs)
59 except BaseException as exc:
60 self.future.set_exception(exc)

File ~\miniconda3\lib\site-packages\intake_esm\core.py:789, in _load_source(key, source)
788 def _load_source(key, source):
--> 789 return key, source.to_dask()

File ~\miniconda3\lib\site-packages\intake_esm\source.py:273, in ESMDataSource.to_dask(self)
271 def to_dask(self):
272 """Return xarray object (which will have chunks)"""
--> 273 self._load_metadata()
274 return self._ds

File ~\miniconda3\lib\site-packages\intake\source\base.py:285, in DataSourceBase._load_metadata(self)
283 """load metadata only if needed"""
284 if self._schema is None:
--> 285 self._schema = self._get_schema()
286 self.dtype = self._schema.dtype
287 self.shape = self._schema.shape

File ~\miniconda3\lib\site-packages\intake_esm\source.py:205, in ESMDataSource._get_schema(self)
202 def _get_schema(self) -> Schema:
204 if self._ds is None:
--> 205 self._open_dataset()
206 metadata = {'dims': {}, 'data_vars': {}, 'coords': ()}
207 self._schema = Schema(
208 datashape=None,
209 dtype=None,
(...)
212 extra_metadata=metadata,
213 )

File ~\miniconda3\lib\site-packages\intake_esm\source.py:265, in ESMDataSource._open_dataset(self)
262 self._ds.attrs[OPTIONS['dataset_key']] = self.key
264 except Exception as exc:
--> 265 raise ESMDataSourceError(
266 f"""Failed to load dataset with key='{self.key}'
267 You can use cat['{self.key}'].df to inspect the assets/files for this key.
268 """
269 ) from exc

ESMDataSourceError: Failed to load dataset with key='CMIP.IPSL.IPSL-CM6A-LR.historical.Oyr.gn'
You can use cat['CMIP.IPSL.IPSL-CM6A-LR.historical.Oyr.gn'].df to inspect the assets/files for this key.

Could someone help me out? I tried other subset. Same errror raised.

[andersy005]

@zhentaom4w, it appears you are missing the packages needed to interact with Google cloud storage. You can install these via pip or conda

$ conda install -c conda-forge gcsfs

Or

$ python -m pip install gcsfs

[zhentaom4w]

Ty @andersy005 It works !