Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

added overwrite parameter reporting #2704

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions deeplake/api/dataset.py
Expand Up @@ -256,6 +256,7 @@ def init(
"lock_timeout": lock_timeout,
"lock_enabled": lock_enabled,
"index_params": index_params,
"overwrite": overwrite,
}

if access_method == "stream":
Expand Down Expand Up @@ -469,6 +470,7 @@ def empty(
"lock_timeout": lock_timeout,
"lock_enabled": lock_enabled,
"index_params": index_params,
"overwrite": overwrite,
}
ret = dataset._load(dataset_kwargs, create=True)
return ret
Expand Down
11 changes: 7 additions & 4 deletions deeplake/core/dataset/dataset.py
Expand Up @@ -178,6 +178,7 @@ def __init__(
view_base: Optional["Dataset"] = None,
libdeeplake_dataset=None,
index_params: Optional[Dict[str, Union[int, str]]] = None,
overwrite: bool = False,
**kwargs,
):
"""Initializes a new or existing dataset.
Expand Down Expand Up @@ -205,7 +206,8 @@ def __init__(
view_base (Optional["Dataset"]): Base dataset of this view.
libdeeplake_dataset : The libdeeplake dataset object corresponding to this dataset.
index_params: (Dict[str, Union[int, str]] Optional) VDB index parameter. Defaults to ``None.``

overwrite (bool): If ``True``, overwrites the dataset if it already exists. Defaults to ``False``.

Raises:
ValueError: If an existing local path is given, it must be a directory.
ImproperDatasetInitialization: Exactly one argument out of 'path' and 'storage' needs to be specified.
Expand Down Expand Up @@ -256,11 +258,12 @@ def __init__(
d["_temp_tensors"] = []
d["_vc_info_updated"] = True
d["_query_string"] = None
d["overwrite"] = overwrite
dct = self.__dict__
dct.update(d)

try:
self._set_derived_attributes(address=address)
self._set_derived_attributes(address=address, overwrite=overwrite)
except LockedException:
raise LockedException(
"This dataset cannot be open for writing as it is locked by another machine. Try loading the dataset with `read_only=True`."
Expand Down Expand Up @@ -2405,10 +2408,10 @@ def _get_total_meta(self):
}

def _set_derived_attributes(
self, verbose: bool = True, address: Optional[str] = None
self, verbose: bool = True, address: Optional[str] = None, overwrite: bool = False,
):
"""Sets derived attributes during init and unpickling."""
if self.is_first_load:
if self.is_first_load or overwrite:
self.storage.autoflush = True
self._load_version_info(address)
self._load_link_creds()
Expand Down
1 change: 1 addition & 0 deletions deeplake/core/dataset/deeplake_cloud_dataset.py
Expand Up @@ -109,6 +109,7 @@ def _send_event(
"commit_id": self.commit_id,
"pending_commit_id": self.pending_commit_id,
"has_head_changes": has_head_changes,
"overwrite": self.overwrite,
}
deeplake_meta.update(common_meta)
event_dict = {
Expand Down