Merge pull request #2745 from activeloopai/fy_tfm_opt

Transform speedup and bugfix
activeloopai · Jan 19, 2024 · 25b688a · 25b688a
2 parents 3c5d48a + 6b5d4d7
commit 25b688a
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/deeplake/core/chunk/chunk_compressed_chunk.py b/deeplake/core/chunk/chunk_compressed_chunk.py
@@ -151,6 +151,9 @@ def extend_if_has_space_byte_compression_numpy(
         update_tensor_meta: bool = True,
     ):
         sample = incoming_samples[0]
+        shape = sample.shape
+        if not shape:
+            shape = (1,)
         chunk_dtype = self.dtype
         sample_dtype = sample.dtype
         if chunk_dtype == sample_dtype:
@@ -220,7 +223,7 @@ def extend_if_has_space_byte_compression_numpy(
         if num_samples:
             self.register_in_meta_and_headers(
                 sample_nbytes,
-                sample.shape,
+                shape,
                 update_tensor_meta=update_tensor_meta,
                 num_samples=num_samples,
             )

diff --git a/deeplake/core/transform/transform_dataset.py b/deeplake/core/transform/transform_dataset.py
@@ -237,7 +237,10 @@ def flush(self, clear_on_fail=True):
                     name = self._get_engine_name(name)
                     updated_tensors[name] = 0
                     chunk_engine = all_chunk_engines[name]
-                    callback = chunk_engine._transform_callback
+                    if chunk_engine.tensor_meta.links:
+                        callback = chunk_engine._transform_callback
+                    else:
+                        callback = None
 
                     meta = chunk_engine.tensor_meta
                     if meta.length == 0 and meta.dtype is None: