Merge pull request #2702 from activeloopai/fy_sup_len_warn

Avoid `len(ds)` warnings internally
activeloopai · Nov 29, 2023 · 9d798f1 · 9d798f1
2 parents eb3a2ac + 572a026
commit 9d798f1
Showing 1 changed file with 9 additions and 7 deletions.
diff --git a/deeplake/core/dataset/dataset.py b/deeplake/core/dataset/dataset.py
@@ -1355,7 +1355,7 @@ def __setattr__(self, name: str, value):
 
     def __iter__(self):
         dataset_read(self)
-        for i in range(len(self)):
+        for i in range(self.__len__(warn=False)):
             yield self.__getitem__(
                 i, is_iteration=not isinstance(self.index.values[0], list)
             )
@@ -2150,7 +2150,9 @@ def pytorch(
         )
 
         if progressbar:
-            dataloader = tqdm(dataloader, desc=self.path, total=len(self) // batch_size)
+            dataloader = tqdm(
+                dataloader, desc=self.path, total=self.__len__(warn=False) // batch_size
+            )
         dataset_read(self)
         return dataloader
 
@@ -3054,7 +3056,6 @@ def _append_or_extend(
 
         """
         tensors = self.tensors
-        new_row_ids = list(range(len(self), len(self) + len(sample)))
         if isinstance(sample, Dataset):
             sample = sample.tensors
         if not isinstance(sample, dict):
@@ -3177,7 +3178,8 @@ def extend(
                 raise ValueError(
                     f"Incoming samples are not of equal lengths. Incoming sample sizes: {sizes}"
                 )
-        new_row_ids = list(range(len(self), len(self) + n))
+        len_ds = self.__len__(warn=False)
+        new_row_ids = list(range(len_ds, len_ds + n))
         [f() for f in list(self._update_hooks.values())]
         if extend:
             if ignore_errors:
@@ -3247,7 +3249,7 @@ def append(
             >>> ds.append({"data": [1, 2, 3, 4], "labels":[0, 1, 2, 3]})
 
         """
-        new_row_ids = [len(self)]
+        new_row_ids = [self.__len__(warn=False)]
         self._append_or_extend(
             sample,
             extend=False,
@@ -3344,7 +3346,7 @@ def get_sample_from_engine(
                 index_maintenance.index_operation_dataset(
                     self,
                     dml_type=_INDEX_OPERATION_MAPPING["UPDATE"],
-                    rowids=list(self.index.values[0].indices(len(self))),
+                    rowids=list(self.index.values[0].indices(self.__len__(warn=False))),
                 )
             except Exception as e:
                 for k, v in saved.items():
@@ -3361,7 +3363,7 @@ def get_sample_from_engine(
                 index_maintenance.index_operation_dataset(
                     self,
                     dml_type=_INDEX_OPERATION_MAPPING["UPDATE"],
-                    rowids=list(self.index.values[0].indices(len(self))),
+                    rowids=list(self.index.values[0].indices(self.__len__(warn=False))),
                 )
                 raise e
             finally: