force cleanup is RAM is filled more than 95%

deeplake/core/storage/lru_cache.py

@@ -4,6 +4,7 @@
 from deeplake.core.storage.deeplake_memory_object import DeepLakeMemoryObject
 from deeplake.core.chunk.base_chunk import BaseChunk
 from typing import Any, Dict, Optional, Union
+import psutil
 
 from deeplake.core.storage.provider import StorageProvider
 import time
@@ -415,7 +416,10 @@ def _free_up_space(self, extra_size: int):
         Args:
             extra_size (int): the space that needs is required in bytes.
         """
-        while self.cache_used > 0 and extra_size + self.cache_used > self.cache_size:
+        while self.cache_used > 0 and (
+            extra_size + self.cache_used > self.cache_size
+            or psutil.virtual_memory().percent > 95
+        ):
             self._pop_from_cache()
 
     def _pop_from_cache(self):

deeplake/requirements/common.txt

@@ -24,3 +24,4 @@ azure-cli
 azure-identity
 azure-storage-blob
 pydantic
+psutil