Merge pull request #2515 from activeloopai/fy_fix_transform

[AL-2353] Add delay for transform progress bar updates
activeloopai · Aug 4, 2023 · ac76dea · ac76dea
2 parents 5792546 + 5e38cbb
commit ac76dea
Showing 1 changed file with 12 additions and 1 deletion.
diff --git a/deeplake/util/transform.py b/deeplake/util/transform.py
@@ -202,6 +202,9 @@ def _transform_and_append_data_slice(
 
     pipeline_checked = False
 
+    last_pg_update_time = time.time()
+    progress = 0
+
     for i, sample in enumerate(
         (data_slice[i : i + 1] for i in range(n))
         if pd and isinstance(data_slice, pd.DataFrame)
@@ -237,7 +240,15 @@ def _transform_and_append_data_slice(
                     skipped_samples_in_current_batch = 0
 
                 if pg_callback is not None:
-                    pg_callback(1)
+                    progress += 1
+                    if (
+                        time.time() - last_pg_update_time
+                        > TRANSFORM_PROGRESSBAR_UPDATE_INTERVAL
+                        or i == n - 1
+                    ):
+                        pg_callback(progress)
+                        progress = 0
+                        last_pg_update_time = time.time()
 
         # failure at chunk_engine
         # retry one sample at a time