Skip to content

Commit

Permalink
Use libdeeplake batch requests for getting bytes. (#2783)
Browse files Browse the repository at this point in the history
* Switch to batch request for indra tensor bytes.

* Bump libdeeplake version.

* Fixed tests.
  • Loading branch information
khustup2 committed Feb 26, 2024
1 parent 17442e3 commit aacf27a
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 10 deletions.
19 changes: 10 additions & 9 deletions deeplake/core/dataset/deeplake_query_tensor.py
Expand Up @@ -78,20 +78,21 @@ def numpy(

def text(self, fetch_chunks: bool = False):
"""Return text data. Only applicable for tensors with 'text' base htype."""
bs = self.indra_tensor.bytes()
if self.ndim == 1:
return self.indra_tensor.bytes().decode()
return list(
self.indra_tensor[i].bytes().decode() for i in range(len(self.indra_tensor))
)
return bs.decode()
if isinstance(bs, bytes):
return [bs.decode()]
return list(b.decode() for b in bs)

def dict(self, fetch_chunks: bool = False):
"""Return json data. Only applicable for tensors with 'json' base htype."""
bs = self.indra_tensor.bytes()
if self.ndim == 1:
return json.loads(self.indra_tensor.bytes().decode())
return list(
json.loads(self.indra_tensor[i].bytes().decode())
for i in range(len(self.indra_tensor))
)
return json.loads(bs.decode())
if isinstance(bs, bytes):
return [json.loads(bs.decode())]
return list(json.loads(b.decode()) for b in self.indra_tensor.bytes())

@property
def dtype(self):
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Expand Up @@ -70,7 +70,7 @@ def libdeeplake_available():
extras_require["all"] = [req_map[r] for r in all_extras]

if libdeeplake_available():
libdeeplake = "libdeeplake==0.0.101"
libdeeplake = "libdeeplake==0.0.104"
extras_require["enterprise"] = [libdeeplake, "pyjwt"]
extras_require["all"].append(libdeeplake)
install_requires.append(libdeeplake)
Expand Down

0 comments on commit aacf27a

Please sign in to comment.