test_hierarchy.py

import atexit
import operator
import os
import sys
import pickle
import shutil
import tempfile
import textwrap
import unittest

import numpy as np
import pytest

try:
    import ipytree
except ImportError:  # pragma: no cover
    ipytree = None

from numcodecs import Zlib
from numpy.testing import assert_array_equal

from zarr._storage.store import _get_metadata_suffix, v3_api_available
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import open_array
from zarr.hierarchy import Group, group, open_group
from zarr.storage import (
    ABSStore,
    DBMStore,
    KVStore,
    DirectoryStore,
    FSStore,
    LMDBStore,
    LRUStoreCache,
    MemoryStore,
    NestedDirectoryStore,
    SQLiteStore,
    ZipStore,
    array_meta_key,
    atexit_rmglob,
    atexit_rmtree,
    data_root,
    group_meta_key,
    init_array,
    init_group,
    meta_root,
)
from zarr._storage.v3 import (
    ABSStoreV3,
    KVStoreV3,
    DirectoryStoreV3,
    MemoryStoreV3,
    FSStoreV3,
    ZipStoreV3,
    DBMStoreV3,
    LMDBStoreV3,
    SQLiteStoreV3,
    LRUStoreCacheV3,
)
from zarr.util import InfoReporter, buffer_size
from zarr.tests.util import skip_test_env_var, have_fsspec, abs_container, mktemp


_VERSIONS = (2, 3) if v3_api_available else (2,)

# noinspection PyStatementEffect


class TestGroup(unittest.TestCase):
    @staticmethod
    def create_store():
        # can be overridden in sub-classes
        return KVStore(dict()), None

    def create_group(
        self, store=None, path=None, read_only=False, chunk_store=None, synchronizer=None
    ):
        # can be overridden in sub-classes
        if store is None:
            store, chunk_store = self.create_store()
        init_group(store, path=path, chunk_store=chunk_store)
        g = Group(
            store,
            path=path,
            read_only=read_only,
            chunk_store=chunk_store,
            synchronizer=synchronizer,
        )
        return g

    def test_ipython_repr_methods(self):
        g = self.create_group()
        for method in [
            "html",
            "json",
            "javascript",
            "markdown",
            "svg",
            "png",
            "jpeg",
            "latex",
            "pdf",
            "mimebundle",
        ]:
            assert operator.methodcaller(f"_repr_{method}_")(g) is None
        with pytest.raises(AttributeError):
            g._ipython_display_()
        with pytest.raises(AttributeError):
            g._ipython_canary_method_should_not_exist_()

    def test_group_init_1(self):
        store, chunk_store = self.create_store()
        g = self.create_group(store, chunk_store=chunk_store)
        assert store is g.store
        if chunk_store is None:
            assert store is g.chunk_store
        else:
            assert chunk_store is g.chunk_store
        assert not g.read_only
        assert "" == g.path
        assert "/" == g.name
        assert "" == g.basename
        assert isinstance(g.attrs, Attributes)
        g.attrs["foo"] = "bar"
        assert g.attrs["foo"] == "bar"
        assert isinstance(g.info, InfoReporter)
        assert isinstance(repr(g.info), str)
        assert isinstance(g.info._repr_html_(), str)
        store.close()

    def test_group_init_2(self):
        store, chunk_store = self.create_store()
        g = self.create_group(store, chunk_store=chunk_store, path="/foo/bar/", read_only=True)
        assert store is g.store
        assert g.read_only
        assert "foo/bar" == g.path
        assert "/foo/bar" == g.name
        assert "bar" == g.basename
        assert isinstance(g.attrs, Attributes)
        store.close()

    def test_group_init_errors_1(self):
        store, chunk_store = self.create_store()
        # group metadata not initialized
        with pytest.raises(ValueError):
            Group(store, chunk_store=chunk_store)
        store.close()

    def test_group_repr(self):
        store, chunk_store = self.create_store()
        g = self.create_group(store, chunk_store=chunk_store)
        assert g.name in repr(g)

    def test_group_init_errors_2(self):
        store, chunk_store = self.create_store()
        init_array(store, shape=1000, chunks=100, chunk_store=chunk_store)
        # array blocks group
        with pytest.raises(ValueError):
            Group(store, chunk_store=chunk_store)
        store.close()

    def _subgroup_path(self, group, path):
        path = path.rstrip("/")
        group_path = "/".join([group.path, path])
        group_path = group_path.lstrip("/")
        group_name = "/" + group_path
        return group_path, group_name

    def test_create_group(self):
        g1 = self.create_group()

        if g1._version == 2:
            path, name = "", "/"
        else:
            path, name = "group", "/group"
        # check root group
        assert path == g1.path
        assert name == g1.name

        # create level 1 child group
        g2 = g1.create_group("foo")
        # check with relative path
        path, name = self._subgroup_path(g1, "foo")
        assert isinstance(g2, Group)
        assert path == g2.path
        assert name == g2.name

        # create level 2 child group
        g3 = g2.create_group("bar")
        path, name = self._subgroup_path(g2, "bar")
        assert isinstance(g3, Group)
        assert path == g3.path
        assert name == g3.name

        # create level 3 child group
        g4 = g1.create_group("foo/bar/baz")
        path, name = self._subgroup_path(g1, "foo/bar/baz")
        assert isinstance(g4, Group)
        assert path == g4.path
        assert name == g4.name

        # create level 3 group via root
        g5 = g4.create_group("/a/b/c/")
        assert isinstance(g5, Group)
        assert "a/b/c" == g5.path
        assert "/a/b/c" == g5.name

        # test non-str keys
        class Foo:
            def __init__(self, s):
                self.s = s

            def __str__(self):
                return self.s

        o = Foo("test/object")
        go = g1.create_group(o)
        path, name = self._subgroup_path(g1, str(o))
        assert isinstance(go, Group)
        assert path == go.path
        go = g1.create_group(b"test/bytes")
        path, name = self._subgroup_path(g1, "test/bytes")
        assert isinstance(go, Group)
        assert path == go.path

        # test bad keys
        with pytest.raises(ValueError):
            g1.create_group("foo")  # already exists
        if g1._version == 2:
            with pytest.raises(ValueError):
                g1.create_group("a/b/c")  # already exists
        elif g1._version == 3:
            # for v3 'group/a/b/c' does not already exist
            g1.create_group("a/b/c")
        with pytest.raises(ValueError):
            g4.create_group("/a/b/c")  # already exists
        with pytest.raises(ValueError):
            g1.create_group("")

        # multi
        g6, g7 = g1.create_groups("y", "z")
        assert isinstance(g6, Group)
        assert g6.path == self._subgroup_path(g1, "y")[0]
        assert isinstance(g7, Group)
        assert g7.path == self._subgroup_path(g1, "z")[0]

        g1.store.close()

    def test_require_group(self):
        g1 = self.create_group()

        # test creation
        g2 = g1.require_group("foo")
        path, name = self._subgroup_path(g1, "foo")
        assert isinstance(g2, Group)
        assert path == g2.path
        g3 = g2.require_group("bar")
        path, name = self._subgroup_path(g2, "bar")
        assert isinstance(g3, Group)
        assert path == g3.path
        g4 = g1.require_group("foo/bar/baz")
        path, name = self._subgroup_path(g1, "foo/bar/baz")
        assert isinstance(g4, Group)
        assert path == g4.path
        g5 = g4.require_group("/a/b/c/")
        assert isinstance(g5, Group)
        assert "a/b/c" == g5.path

        # test when already created
        g2a = g1.require_group("foo")
        assert g2 == g2a
        assert g2.store is g2a.store
        g3a = g2a.require_group("bar")
        assert g3 == g3a
        assert g3.store is g3a.store
        g4a = g1.require_group("foo/bar/baz")
        assert g4 == g4a
        assert g4.store is g4a.store
        g5a = g4a.require_group("/a/b/c/")
        assert g5 == g5a
        assert g5.store is g5a.store

        # test path normalization
        if g1._version == 2:
            assert g1.require_group("quux") == g1.require_group("/quux/")
        elif g1._version:
            # These are not equal in v3!
            # 'quux' will be within the group:
            #      meta/root/group/quux.group.json
            # '/quux/' will be outside of the group at:
            #      meta/root/quux.group.json
            assert g1.require_group("quux") != g1.require_group("/quux/")

        # multi
        g6, g7 = g1.require_groups("y", "z")
        assert isinstance(g6, Group)
        assert g6.path == self._subgroup_path(g1, "y")[0]
        assert isinstance(g7, Group)
        assert g7.path == self._subgroup_path(g1, "z")[0]

        g1.store.close()

    def test_rmdir_group_and_array_metadata_files(self):
        """Test group.store's rmdir method.

        This test case was added to complete test coverage of `ABSStore.rmdir`.
        """
        g1 = self.create_group()
        # create a dataset
        g1.create_dataset("arr1", shape=(100,), chunks=(10,), dtype=np.uint8)

        # create level 1 child group
        g2 = g1.create_group("foo")
        g1.create_dataset("arr2", shape=(100,), chunks=(10,), dtype=np.uint8)

        if g1._version > 2 and g1.store.is_erasable():
            arr_path = g1.path + "/arr1"
            sfx = _get_metadata_suffix(g1.store)
            array_meta_file = meta_root + arr_path + ".array" + sfx
            assert array_meta_file in g1.store
            group_meta_file = meta_root + g2.path + ".group" + sfx
            assert group_meta_file in g1.store

            # rmdir on the array path should also remove the metadata file
            g1.store.rmdir(arr_path)
            assert array_meta_file not in g1.store
            # rmdir on the group path should also remove its metadata file
            g1.store.rmdir(g2.path)
            assert group_meta_file not in g1.store

    def _dataset_path(self, group, path):
        path = path.rstrip("/")
        absolute = path.startswith("/")
        if absolute:
            dataset_path = path
        else:
            dataset_path = "/".join([group.path, path])
        dataset_path = dataset_path.lstrip("/")
        dataset_name = "/" + dataset_path
        return dataset_path, dataset_name

    def test_create_dataset(self):
        g = self.create_group()

        # create as immediate child
        dpath = "foo"
        d1 = g.create_dataset(dpath, shape=1000, chunks=100)
        path, name = self._dataset_path(g, dpath)
        assert isinstance(d1, Array)
        assert (1000,) == d1.shape
        assert (100,) == d1.chunks
        assert path == d1.path
        assert name == d1.name
        assert g.store is d1.store

        # create as descendant
        dpath = "/a/b/c/"
        d2 = g.create_dataset(
            dpath,
            shape=2000,
            chunks=200,
            dtype="i1",
            compression="zlib",
            compression_opts=9,
            fill_value=42,
            order="F",
        )
        path, name = self._dataset_path(g, dpath)
        assert isinstance(d2, Array)
        assert (2000,) == d2.shape
        assert (200,) == d2.chunks
        assert np.dtype("i1") == d2.dtype
        assert "zlib" == d2.compressor.codec_id
        assert 9 == d2.compressor.level
        assert 42 == d2.fill_value
        assert "F" == d2.order
        assert path == d2.path
        assert name == d2.name
        assert g.store is d2.store

        # create with data
        data = np.arange(3000, dtype="u2")
        dpath = "bar"
        d3 = g.create_dataset(dpath, data=data, chunks=300)
        path, name = self._dataset_path(g, dpath)
        assert isinstance(d3, Array)
        assert (3000,) == d3.shape
        assert (300,) == d3.chunks
        assert np.dtype("u2") == d3.dtype
        assert_array_equal(data, d3[:])
        assert path == d3.path
        assert name == d3.name
        assert g.store is d3.store

        # compression arguments handling follows...

        # compression_opts as dict
        d = g.create_dataset(
            "aaa",
            shape=1000,
            dtype="u1",
            compression="blosc",
            compression_opts=dict(cname="zstd", clevel=1, shuffle=2),
        )
        assert d.compressor.codec_id == "blosc"
        assert "zstd" == d.compressor.cname
        assert 1 == d.compressor.clevel
        assert 2 == d.compressor.shuffle

        # compression_opts as sequence
        d = g.create_dataset(
            "bbb", shape=1000, dtype="u1", compression="blosc", compression_opts=("zstd", 1, 2)
        )
        assert d.compressor.codec_id == "blosc"
        assert "zstd" == d.compressor.cname
        assert 1 == d.compressor.clevel
        assert 2 == d.compressor.shuffle

        # None compression_opts
        d = g.create_dataset("ccc", shape=1000, dtype="u1", compression="zlib")
        assert d.compressor.codec_id == "zlib"
        assert 1 == d.compressor.level

        # None compression
        d = g.create_dataset("ddd", shape=1000, dtype="u1", compression=None)
        assert d.compressor is None

        # compressor as compression
        d = g.create_dataset("eee", shape=1000, dtype="u1", compression=Zlib(1))
        assert d.compressor.codec_id == "zlib"
        assert 1 == d.compressor.level

        g.store.close()

    def test_require_dataset(self):
        g = self.create_group()

        # create
        dpath = "foo"
        d1 = g.require_dataset(dpath, shape=1000, chunks=100, dtype="f4")
        d1[:] = np.arange(1000)
        path, name = self._dataset_path(g, dpath)
        assert isinstance(d1, Array)
        assert (1000,) == d1.shape
        assert (100,) == d1.chunks
        assert np.dtype("f4") == d1.dtype
        assert path == d1.path
        assert name == d1.name
        assert g.store is d1.store
        assert_array_equal(np.arange(1000), d1[:])

        # require
        d2 = g.require_dataset(dpath, shape=1000, chunks=100, dtype="f4")
        assert isinstance(d2, Array)
        assert (1000,) == d2.shape
        assert (100,) == d2.chunks
        assert np.dtype("f4") == d2.dtype
        assert path == d2.path
        assert name == d2.name
        assert g.store is d2.store
        assert_array_equal(np.arange(1000), d2[:])
        assert d1 == d2

        # bad shape - use TypeError for h5py compatibility
        with pytest.raises(TypeError):
            g.require_dataset("foo", shape=2000, chunks=100, dtype="f4")

        # dtype matching
        # can cast
        d3 = g.require_dataset("foo", shape=1000, chunks=100, dtype="i2")
        assert np.dtype("f4") == d3.dtype
        assert d1 == d3
        with pytest.raises(TypeError):
            # cannot cast
            g.require_dataset("foo", shape=1000, chunks=100, dtype="i4")
        with pytest.raises(TypeError):
            # can cast but not exact match
            g.require_dataset("foo", shape=1000, chunks=100, dtype="i2", exact=True)

        g.store.close()

    def test_create_errors(self):
        g = self.create_group()

        # array obstructs group, array
        g.create_dataset("foo", shape=100, chunks=10)
        with pytest.raises(ValueError):
            g.create_group("foo/bar")
        with pytest.raises(ValueError):
            g.require_group("foo/bar")
        with pytest.raises(ValueError):
            g.create_dataset("foo/bar", shape=100, chunks=10)
        with pytest.raises(ValueError):
            g.require_dataset("foo/bar", shape=100, chunks=10)

        # array obstructs group, array
        g.create_dataset("a/b", shape=100, chunks=10)
        with pytest.raises(ValueError):
            g.create_group("a/b")
        with pytest.raises(ValueError):
            g.require_group("a/b")
        with pytest.raises(ValueError):
            g.create_dataset("a/b", shape=100, chunks=10)

        # group obstructs array
        g.create_group("c/d")
        with pytest.raises(ValueError):
            g.create_dataset("c", shape=100, chunks=10)
        with pytest.raises(ValueError):
            g.require_dataset("c", shape=100, chunks=10)
        with pytest.raises(ValueError):
            g.create_dataset("c/d", shape=100, chunks=10)
        with pytest.raises(ValueError):
            g.require_dataset("c/d", shape=100, chunks=10)

        # h5py compatibility, accept 'fillvalue'
        d = g.create_dataset("x", shape=100, chunks=10, fillvalue=42)
        assert 42 == d.fill_value

        # h5py compatibility, ignore 'shuffle'
        with pytest.warns(UserWarning, match="ignoring keyword argument 'shuffle'"):
            g.create_dataset("y", shape=100, chunks=10, shuffle=True)

        # read-only
        g = self.create_group(read_only=True)
        with pytest.raises(PermissionError):
            g.create_group("zzz")
        with pytest.raises(PermissionError):
            g.require_group("zzz")
        with pytest.raises(PermissionError):
            g.create_dataset("zzz", shape=100, chunks=10)
        with pytest.raises(PermissionError):
            g.require_dataset("zzz", shape=100, chunks=10)

        g.store.close()

    def test_create_overwrite(self):
        try:
            for method_name in "create_dataset", "create", "empty", "zeros", "ones":
                g = self.create_group()
                getattr(g, method_name)("foo", shape=100, chunks=10)

                # overwrite array with array
                d = getattr(g, method_name)("foo", shape=200, chunks=20, overwrite=True)
                assert (200,) == d.shape
                # overwrite array with group
                g2 = g.create_group("foo", overwrite=True)
                assert 0 == len(g2)
                # overwrite group with array
                d = getattr(g, method_name)("foo", shape=300, chunks=30, overwrite=True)
                assert (300,) == d.shape
                # overwrite array with group
                d = getattr(g, method_name)("foo/bar", shape=400, chunks=40, overwrite=True)
                assert (400,) == d.shape
                assert isinstance(g["foo"], Group)

                g.store.close()
        except NotImplementedError:
            pass

    def test_getitem_contains_iterators(self):
        # setup
        g1 = self.create_group()
        g2 = g1.create_group("foo/bar")
        if g1._version == 2:
            d1 = g2.create_dataset("/a/b/c", shape=1000, chunks=100)
        else:
            # v3: cannot create a dataset at the root by starting with /
            #     instead, need to create the dataset on g1 directly
            d1 = g1.create_dataset("a/b/c", shape=1000, chunks=100)
        d1[:] = np.arange(1000)
        d2 = g1.create_dataset("foo/baz", shape=3000, chunks=300)
        d2[:] = np.arange(3000)

        # test __getitem__
        assert isinstance(g1["foo"], Group)
        assert isinstance(g1["foo"]["bar"], Group)
        assert isinstance(g1["foo/bar"], Group)
        if g1._version == 2:
            assert isinstance(g1["/foo/bar/"], Group)
        else:
            # start or end with / raises KeyError
            # TODO: should we allow stripping of these on v3?
            with pytest.raises(KeyError):
                assert isinstance(g1["/foo/bar/"], Group)
        assert isinstance(g1["foo/baz"], Array)
        assert g2 == g1["foo/bar"]
        assert g1["foo"]["bar"] == g1["foo/bar"]
        assert d2 == g1["foo/baz"]
        assert_array_equal(d2[:], g1["foo/baz"])
        assert isinstance(g1["a"], Group)
        assert isinstance(g1["a"]["b"], Group)
        assert isinstance(g1["a/b"], Group)
        assert isinstance(g1["a"]["b"]["c"], Array)
        assert isinstance(g1["a/b/c"], Array)
        assert d1 == g1["a/b/c"]
        assert g1["a"]["b"]["c"] == g1["a/b/c"]
        assert_array_equal(d1[:], g1["a/b/c"][:])

        # test __contains__
        assert "foo" in g1
        assert "foo/bar" in g1
        assert "foo/baz" in g1
        assert "bar" in g1["foo"]
        assert "a" in g1
        assert "a/b" in g1
        assert "a/b/c" in g1
        assert "baz" not in g1
        assert "a/b/c/d" not in g1
        assert "a/z" not in g1
        assert "quux" not in g1["foo"]

        # test key errors
        with pytest.raises(KeyError):
            g1["baz"]
        with pytest.raises(KeyError):
            g1["x/y/z"]

        # test __len__
        assert 2 == len(g1)
        assert 2 == len(g1["foo"])
        assert 0 == len(g1["foo/bar"])
        assert 1 == len(g1["a"])
        assert 1 == len(g1["a/b"])

        # test __iter__, keys()

        if g1._version == 2:
            # currently assumes sorted by key
            assert ["a", "foo"] == list(g1)
            assert ["a", "foo"] == list(g1.keys())
            assert ["bar", "baz"] == list(g1["foo"])
            assert ["bar", "baz"] == list(g1["foo"].keys())
        else:
            # v3 is not necessarily sorted by key
            assert ["a", "foo"] == sorted(list(g1))
            assert ["a", "foo"] == sorted(list(g1.keys()))
            assert ["bar", "baz"] == sorted(list(g1["foo"]))
            assert ["bar", "baz"] == sorted(list(g1["foo"].keys()))
        assert [] == sorted(g1["foo/bar"])
        assert [] == sorted(g1["foo/bar"].keys())

        # test items(), values()
        # currently assumes sorted by key

        items = list(g1.items())
        values = list(g1.values())
        if g1._version == 3:
            # v3 are not automatically sorted by key
            items, values = zip(*sorted(zip(items, values), key=lambda x: x[0]))
        assert "a" == items[0][0]
        assert g1["a"] == items[0][1]
        assert g1["a"] == values[0]
        assert "foo" == items[1][0]
        assert g1["foo"] == items[1][1]
        assert g1["foo"] == values[1]

        items = list(g1["foo"].items())
        values = list(g1["foo"].values())
        if g1._version == 3:
            # v3 are not automatically sorted by key
            items, values = zip(*sorted(zip(items, values), key=lambda x: x[0]))
        assert "bar" == items[0][0]
        assert g1["foo"]["bar"] == items[0][1]
        assert g1["foo"]["bar"] == values[0]
        assert "baz" == items[1][0]
        assert g1["foo"]["baz"] == items[1][1]
        assert g1["foo"]["baz"] == values[1]

        # test array_keys(), arrays(), group_keys(), groups()

        groups = list(g1.groups())
        arrays = list(g1.arrays())
        if g1._version == 2:
            # currently assumes sorted by key
            assert ["a", "foo"] == list(g1.group_keys())
        else:
            assert ["a", "foo"] == sorted(list(g1.group_keys()))
            groups = sorted(groups)
            arrays = sorted(arrays)
        assert "a" == groups[0][0]
        assert g1["a"] == groups[0][1]
        assert "foo" == groups[1][0]
        assert g1["foo"] == groups[1][1]
        assert [] == list(g1.array_keys())
        assert [] == arrays

        assert ["bar"] == list(g1["foo"].group_keys())
        assert ["baz"] == list(g1["foo"].array_keys())
        groups = list(g1["foo"].groups())
        arrays = list(g1["foo"].arrays())
        if g1._version == 3:
            groups = sorted(groups)
            arrays = sorted(arrays)
        assert "bar" == groups[0][0]
        assert g1["foo"]["bar"] == groups[0][1]
        assert "baz" == arrays[0][0]
        assert g1["foo"]["baz"] == arrays[0][1]

        # visitor collection tests
        items = []

        def visitor2(obj):
            items.append(obj.path)

        # noinspection PyUnusedLocal
        def visitor3(name, obj=None):
            items.append(name)

        def visitor4(name, obj):
            items.append((name, obj))

        del items[:]
        g1.visitvalues(visitor2)
        expected_items = [
            "a",
            "a/b",
            "a/b/c",
            "foo",
            "foo/bar",
            "foo/baz",
        ]
        if g1._version == 3:
            expected_items = [g1.path + "/" + i for i in expected_items]
        assert expected_items == items

        del items[:]
        g1["foo"].visitvalues(visitor2)
        expected_items = [
            "foo/bar",
            "foo/baz",
        ]
        if g1._version == 3:
            expected_items = [g1.path + "/" + i for i in expected_items]
        assert expected_items == items

        del items[:]
        g1.visit(visitor3)
        assert [
            "a",
            "a/b",
            "a/b/c",
            "foo",
            "foo/bar",
            "foo/baz",
        ] == items

        del items[:]
        g1["foo"].visit(visitor3)
        assert [
            "bar",
            "baz",
        ] == items

        del items[:]
        g1.visitkeys(visitor3)
        assert [
            "a",
            "a/b",
            "a/b/c",
            "foo",
            "foo/bar",
            "foo/baz",
        ] == items

        del items[:]
        g1["foo"].visitkeys(visitor3)
        assert [
            "bar",
            "baz",
        ] == items

        del items[:]
        g1.visititems(visitor3)
        assert [
            "a",
            "a/b",
            "a/b/c",
            "foo",
            "foo/bar",
            "foo/baz",
        ] == items

        del items[:]
        g1["foo"].visititems(visitor3)
        assert [
            "bar",
            "baz",
        ] == items

        del items[:]
        g1.visititems(visitor4)
        for n, o in items:
            assert g1[n] == o

        del items[:]
        g1["foo"].visititems(visitor4)
        for n, o in items:
            assert g1["foo"][n] == o

        # visitor filter tests
        # noinspection PyUnusedLocal
        def visitor0(val, *args):
            name = getattr(val, "path", val)
            if name == "a/b/c/d":
                return True  # pragma: no cover

        # noinspection PyUnusedLocal
        def visitor1(val, *args):
            name = getattr(val, "path", val)
            if name.startswith("group/"):
                # strip the group path for v3
                name = name[6:]
            if name == "a/b/c":
                return True

        assert g1.visit(visitor0) is None
        assert g1.visitkeys(visitor0) is None
        assert g1.visitvalues(visitor0) is None
        assert g1.visititems(visitor0) is None
        assert g1.visit(visitor1) is True
        assert g1.visitkeys(visitor1) is True
        assert g1.visitvalues(visitor1) is True
        assert g1.visititems(visitor1) is True

        g1.store.close()

    # regression test for https://github.com/zarr-developers/zarr-python/issues/1228
    def test_double_counting_group_v3(self):
        root_group = self.create_group()
        group_names = ["foo", "foo-", "foo_"]
        for name in group_names:
            sub_group = root_group.create_group(name)
            sub_group.create("bar", shape=10, dtype="i4")
        assert list(root_group.group_keys()) == sorted(group_names)
        assert list(root_group.groups()) == [
            (name, root_group[name]) for name in sorted(group_names)
        ]

    def test_empty_getitem_contains_iterators(self):
        # setup
        g = self.create_group()

        # test
        assert [] == list(g)
        assert [] == list(g.keys())
        assert 0 == len(g)
        assert "foo" not in g

        g.store.close()

    def test_iterators_recurse(self):
        # setup
        g1 = self.create_group()
        g2 = g1.create_group("foo/bar")
        d1 = g2.create_dataset("/a/b/c", shape=1000, chunks=100)
        d1[:] = np.arange(1000)
        d2 = g1.create_dataset("foo/baz", shape=3000, chunks=300)
        d2[:] = np.arange(3000)
        d3 = g2.create_dataset("zab", shape=2000, chunks=200)
        d3[:] = np.arange(2000)

        # test recursive array_keys
        array_keys = list(g1["foo"].array_keys(recurse=False))
        array_keys_recurse = list(g1["foo"].array_keys(recurse=True))
        assert len(array_keys_recurse) > len(array_keys)
        assert sorted(array_keys_recurse) == ["baz", "zab"]

        # test recursive arrays
        arrays = list(g1["foo"].arrays(recurse=False))
        arrays_recurse = list(g1["foo"].arrays(recurse=True))
        assert len(arrays_recurse) > len(arrays)
        assert "zab" == arrays_recurse[0][0]
        assert g1["foo"]["bar"]["zab"] == arrays_recurse[0][1]

        g1.store.close()

    def test_getattr(self):
        # setup
        g1 = self.create_group()
        g2 = g1.create_group("foo")
        g2.create_dataset("bar", shape=100)

        # test
        assert g1["foo"] == g1.foo
        assert g2["bar"] == g2.bar
        # test that hasattr returns False instead of an exception (issue #88)
        assert not hasattr(g1, "unexistingattribute")

        g1.store.close()

    def test_setitem(self):
        g = self.create_group()
        try:
            data = np.arange(100)
            g["foo"] = data
            assert_array_equal(data, g["foo"])
            data = np.arange(200)
            g["foo"] = data
            assert_array_equal(data, g["foo"])
            # 0d array
            g["foo"] = 42
            assert () == g["foo"].shape
            assert 42 == g["foo"][()]
        except NotImplementedError:
            pass
        g.store.close()

    def test_delitem(self):
        g = self.create_group()
        g.create_group("foo")
        g.create_dataset("bar/baz", shape=100, chunks=10)
        assert "foo" in g
        assert "bar" in g
        assert "bar/baz" in g
        try:
            del g["bar"]
            with pytest.raises(KeyError):
                del g["xxx"]
        except NotImplementedError:
            pass
        else:
            assert "foo" in g
            assert "bar" not in g
            assert "bar/baz" not in g
        g.store.close()

    def test_move(self):
        g = self.create_group()

        data = np.arange(100)
        g["boo"] = data

        data = np.arange(100)
        g["foo"] = data

        g.move("foo", "bar")
        assert "foo" not in g
        assert "bar" in g
        assert_array_equal(data, g["bar"])

        g.move("bar", "foo/bar")
        assert "bar" not in g
        assert "foo" in g
        assert "foo/bar" in g
        assert isinstance(g["foo"], Group)
        assert_array_equal(data, g["foo/bar"])

        g.move("foo", "foo2")
        assert "foo" not in g
        assert "foo/bar" not in g
        assert "foo2" in g
        assert "foo2/bar" in g
        assert isinstance(g["foo2"], Group)
        assert_array_equal(data, g["foo2/bar"])

        g2 = g["foo2"]
        g2.move("bar", "/bar")
        assert "foo2" in g
        assert "foo2/bar" not in g
        if g2._version == 2:
            assert "bar" in g
        else:
            # The `g2.move` call above moved bar to meta/root/bar and
            # meta/data/bar. This is outside the `g` group located at
            # /meta/root/group, so bar is no longer within `g`.
            assert "bar" not in g
            assert "meta/root/bar.array.json" in g._store
            if g._chunk_store:
                assert "data/root/bar/c0" in g._chunk_store
            else:
                assert "data/root/bar/c0" in g._store
        assert isinstance(g["foo2"], Group)
        if g2._version == 2:
            assert_array_equal(data, g["bar"])
        else:
            # TODO: How to access element created outside of group.path in v3?
            #       One option is to make a Hierarchy class representing the
            #       root. Currently Group requires specification of `path`,
            #       but the path of the root would be just '' which is not
            #       currently allowed.
            pass

        with pytest.raises(ValueError):
            g2.move("bar", "bar2")

        with pytest.raises(ValueError):
            g.move("bar", "boo")

        g.store.close()

    def test_array_creation(self):
        grp = self.create_group()

        a = grp.create("a", shape=100, chunks=10)
        assert isinstance(a, Array)
        b = grp.empty("b", shape=100, chunks=10)
        assert isinstance(b, Array)
        assert b.fill_value is None
        c = grp.zeros("c", shape=100, chunks=10)
        assert isinstance(c, Array)
        assert 0 == c.fill_value
        d = grp.ones("d", shape=100, chunks=10)
        assert isinstance(d, Array)
        assert 1 == d.fill_value
        e = grp.full("e", shape=100, chunks=10, fill_value=42)
        assert isinstance(e, Array)
        assert 42 == e.fill_value

        f = grp.empty_like("f", a)
        assert isinstance(f, Array)
        assert f.fill_value is None
        g = grp.zeros_like("g", a)
        assert isinstance(g, Array)
        assert 0 == g.fill_value
        h = grp.ones_like("h", a)
        assert isinstance(h, Array)
        assert 1 == h.fill_value
        i = grp.full_like("i", e)
        assert isinstance(i, Array)
        assert 42 == i.fill_value

        j = grp.array("j", data=np.arange(100), chunks=10)
        assert isinstance(j, Array)
        assert_array_equal(np.arange(100), j[:])

        grp.store.close()

        grp = self.create_group(read_only=True)
        with pytest.raises(PermissionError):
            grp.create("aa", shape=100, chunks=10)
        with pytest.raises(PermissionError):
            grp.empty("aa", shape=100, chunks=10)
        with pytest.raises(PermissionError):
            grp.zeros("aa", shape=100, chunks=10)
        with pytest.raises(PermissionError):
            grp.ones("aa", shape=100, chunks=10)
        with pytest.raises(PermissionError):
            grp.full("aa", shape=100, chunks=10, fill_value=42)
        with pytest.raises(PermissionError):
            grp.array("aa", data=np.arange(100), chunks=10)
        with pytest.raises(PermissionError):
            grp.create("aa", shape=100, chunks=10)
        with pytest.raises(PermissionError):
            grp.empty_like("aa", a)
        with pytest.raises(PermissionError):
            grp.zeros_like("aa", a)
        with pytest.raises(PermissionError):
            grp.ones_like("aa", a)
        with pytest.raises(PermissionError):
            grp.full_like("aa", a)

        grp.store.close()

    def test_paths(self):
        g1 = self.create_group()
        g2 = g1.create_group("foo/bar")

        if g1._version == 2:
            assert g1 == g1["/"]
            assert g1 == g1["//"]
            assert g1 == g1["///"]
            assert g1 == g2["/"]
            assert g1 == g2["//"]
            assert g1 == g2["///"]
            assert g2 == g1["foo/bar"]
            assert g2 == g1["/foo/bar"]
            assert g2 == g1["foo/bar/"]
            assert g2 == g1["//foo/bar"]
            assert g2 == g1["//foo//bar//"]
            assert g2 == g1["///foo///bar///"]
            assert g2 == g2["/foo/bar"]
        else:
            # the expected key format gives a match
            assert g2 == g1["foo/bar"]

            # TODO: Should presence of a trailing slash raise KeyError?
            # The spec says "the final character is not a / character"
            # but we currently strip trailing '/' as done for v2.
            assert g2 == g1["foo/bar/"]

            # double slash also currently works (spec doesn't mention this
            # case, but have kept it for v2 behavior compatibility)
            assert g2 == g1["foo//bar"]

            # TODO, root: fix these cases
            # v3: leading / implies we are at the root, not within a group,
            # so these all raise KeyError
            for path in ["/foo/bar", "//foo/bar", "//foo//bar//", "///fooo///bar///"]:
                with pytest.raises(KeyError):
                    g1[path]

        with pytest.raises(ValueError):
            g1["."]
        with pytest.raises(ValueError):
            g1[".."]
        with pytest.raises(ValueError):
            g1["foo/."]
        with pytest.raises(ValueError):
            g1["foo/.."]
        with pytest.raises(ValueError):
            g1["foo/./bar"]
        with pytest.raises(ValueError):
            g1["foo/../bar"]

        g1.store.close()

    def test_pickle(self):
        # setup group
        g = self.create_group()
        d = g.create_dataset("foo/bar", shape=100, chunks=10)
        d[:] = np.arange(100)
        path = g.path
        name = g.name
        n = len(g)
        keys = list(g)

        # round-trip through pickle
        dump = pickle.dumps(g)
        # some stores cannot be opened twice at the same time, need to close
        # store before can round-trip through pickle
        g.store.close()
        g2 = pickle.loads(dump)

        # verify
        assert path == g2.path
        assert name == g2.name
        assert n == len(g2)
        assert keys == list(g2)
        assert isinstance(g2["foo"], Group)
        assert isinstance(g2["foo/bar"], Array)

        g2.store.close()

    def test_context_manager(self):
        with self.create_group() as g:
            d = g.create_dataset("foo/bar", shape=100, chunks=10)
            d[:] = np.arange(100)


@pytest.mark.parametrize("chunk_dict", [False, True])
def test_group_init_from_dict(chunk_dict):
    if chunk_dict:
        store, chunk_store = dict(), dict()
    else:
        store, chunk_store = dict(), None
    init_group(store, path=None, chunk_store=chunk_store)
    g = Group(store, path=None, read_only=False, chunk_store=chunk_store)
    assert store is not g.store
    assert isinstance(g.store, KVStore)
    if chunk_store is None:
        assert g.store is g.chunk_store
    else:
        assert chunk_store is not g.chunk_store


# noinspection PyStatementEffect
@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3(TestGroup, unittest.TestCase):
    @staticmethod
    def create_store():
        # can be overridden in sub-classes
        return KVStoreV3(dict()), None

    def create_group(
        self, store=None, path="group", read_only=False, chunk_store=None, synchronizer=None
    ):
        # can be overridden in sub-classes
        if store is None:
            store, chunk_store = self.create_store()
        init_group(store, path=path, chunk_store=chunk_store)
        g = Group(
            store,
            path=path,
            read_only=read_only,
            chunk_store=chunk_store,
            synchronizer=synchronizer,
        )
        return g

    def test_group_init_1(self):
        store, chunk_store = self.create_store()
        g = self.create_group(store, chunk_store=chunk_store)
        assert store is g.store
        if chunk_store is None:
            assert store is g.chunk_store
        else:
            assert chunk_store is g.chunk_store
        assert not g.read_only
        # different path/name in v3 case
        assert "group" == g.path
        assert "/group" == g.name
        assert "group" == g.basename

        assert isinstance(g.attrs, Attributes)
        g.attrs["foo"] = "bar"
        assert g.attrs["foo"] == "bar"

        assert isinstance(g.info, InfoReporter)
        assert isinstance(repr(g.info), str)
        assert isinstance(g.info._repr_html_(), str)
        store.close()

    def test_group_init_errors_2(self):
        store, chunk_store = self.create_store()
        path = "tmp"
        init_array(store, path=path, shape=1000, chunks=100, chunk_store=chunk_store)
        # array blocks group
        with pytest.raises(ValueError):
            Group(store, path=path, chunk_store=chunk_store)
        store.close()


class TestGroupWithMemoryStore(TestGroup):
    @staticmethod
    def create_store():
        return MemoryStore(), None


# noinspection PyStatementEffect
@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithMemoryStore(TestGroupWithMemoryStore, TestGroupV3):
    @staticmethod
    def create_store():
        return MemoryStoreV3(), None


class TestGroupWithDirectoryStore(TestGroup):
    @staticmethod
    def create_store():
        path = tempfile.mkdtemp()
        atexit.register(atexit_rmtree, path)
        store = DirectoryStore(path)
        return store, None


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithDirectoryStore(TestGroupWithDirectoryStore, TestGroupV3):
    @staticmethod
    def create_store():
        path = tempfile.mkdtemp()
        atexit.register(atexit_rmtree, path)
        store = DirectoryStoreV3(path)
        return store, None


@skip_test_env_var("ZARR_TEST_ABS")
class TestGroupWithABSStore(TestGroup):
    @staticmethod
    def create_store():
        container_client = abs_container()
        store = ABSStore(client=container_client)
        store.rmdir()
        return store, None

    @pytest.mark.skipif(sys.version_info < (3, 7), reason="attr not serializable in py36")
    def test_pickle(self):
        # internal attribute on ContainerClient isn't serializable for py36 and earlier
        super().test_pickle()


@skip_test_env_var("ZARR_TEST_ABS")
@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithABSStore(TestGroupV3):
    @staticmethod
    def create_store():
        container_client = abs_container()
        store = ABSStoreV3(client=container_client)
        store.rmdir()
        return store, None

    @pytest.mark.skipif(sys.version_info < (3, 7), reason="attr not serializable in py36")
    def test_pickle(self):
        # internal attribute on ContainerClient isn't serializable for py36 and earlier
        super().test_pickle()


class TestGroupWithNestedDirectoryStore(TestGroup):
    @staticmethod
    def create_store():
        path = tempfile.mkdtemp()
        atexit.register(atexit_rmtree, path)
        store = NestedDirectoryStore(path)
        return store, None


@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
class TestGroupWithFSStore(TestGroup):
    @staticmethod
    def create_store():
        path = tempfile.mkdtemp()
        atexit.register(atexit_rmtree, path)
        store = FSStore(path)
        return store, None

    def test_round_trip_nd(self):
        data = np.arange(1000).reshape(10, 10, 10)
        name = "raw"

        store, _ = self.create_store()
        f = open_group(store, mode="w")
        f.create_dataset(name, data=data, chunks=(5, 5, 5), compressor=None)
        assert name in f
        h = open_group(store, mode="r")
        np.testing.assert_array_equal(h[name][:], data)


@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithFSStore(TestGroupWithFSStore, TestGroupV3):
    @staticmethod
    def create_store():
        path = tempfile.mkdtemp()
        atexit.register(atexit_rmtree, path)
        store = FSStoreV3(path)
        return store, None

    def test_round_trip_nd(self):
        data = np.arange(1000).reshape(10, 10, 10)
        name = "raw"

        store, _ = self.create_store()
        f = open_group(store, path="group", mode="w")
        f.create_dataset(name, data=data, chunks=(5, 5, 5), compressor=None)
        h = open_group(store, path="group", mode="r")
        np.testing.assert_array_equal(h[name][:], data)

        f = open_group(store, path="group2", mode="w")

        data_size = data.nbytes
        group_meta_size = buffer_size(store[meta_root + "group.group.json"])
        group2_meta_size = buffer_size(store[meta_root + "group2.group.json"])
        array_meta_size = buffer_size(store[meta_root + "group/raw.array.json"])
        assert store.getsize() == data_size + group_meta_size + group2_meta_size + array_meta_size
        # added case with path to complete coverage
        assert store.getsize("group") == data_size + group_meta_size + array_meta_size
        assert store.getsize("group2") == group2_meta_size
        assert store.getsize("group/raw") == data_size + array_meta_size


@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
class TestGroupWithNestedFSStore(TestGroupWithFSStore):
    @staticmethod
    def create_store():
        path = tempfile.mkdtemp()
        atexit.register(atexit_rmtree, path)
        store = FSStore(path, key_separator="/", auto_mkdir=True)
        return store, None

    def test_inconsistent_dimension_separator(self):
        data = np.arange(1000).reshape(10, 10, 10)
        name = "raw"

        store, _ = self.create_store()
        f = open_group(store, mode="w")

        # cannot specify dimension_separator that conflicts with the store
        with pytest.raises(ValueError):
            f.create_dataset(
                name, data=data, chunks=(5, 5, 5), compressor=None, dimension_separator="."
            )


@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithNestedFSStore(TestGroupV3WithFSStore):
    @staticmethod
    def create_store():
        path = tempfile.mkdtemp()
        atexit.register(atexit_rmtree, path)
        store = FSStoreV3(path, key_separator="/", auto_mkdir=True)
        return store, None

    def test_inconsistent_dimension_separator(self):
        data = np.arange(1000).reshape(10, 10, 10)
        name = "raw"

        store, _ = self.create_store()
        f = open_group(store, path="group", mode="w")

        # cannot specify dimension_separator that conflicts with the store
        with pytest.raises(ValueError):
            f.create_dataset(
                name, data=data, chunks=(5, 5, 5), compressor=None, dimension_separator="."
            )


class TestGroupWithZipStore(TestGroup):
    @staticmethod
    def create_store():
        path = mktemp(suffix=".zip")
        atexit.register(os.remove, path)
        store = ZipStore(path)
        return store, None

    def test_context_manager(self):
        with self.create_group() as g:
            store = g.store
            d = g.create_dataset("foo/bar", shape=100, chunks=10)
            d[:] = np.arange(100)

        # Check that exiting the context manager closes the store,
        # and therefore the underlying ZipFile.
        with pytest.raises(ValueError):
            store.zf.extractall()

    def test_move(self):
        # zip store is not erasable (can so far only append to a zip
        # so we can't test for move.
        pass


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithZipStore(TestGroupWithZipStore, TestGroupV3):
    @staticmethod
    def create_store():
        path = mktemp(suffix=".zip")
        atexit.register(os.remove, path)
        store = ZipStoreV3(path)
        return store, None


class TestGroupWithDBMStore(TestGroup):
    @staticmethod
    def create_store():
        path = mktemp(suffix=".anydbm")
        atexit.register(atexit_rmglob, path + "*")
        store = DBMStore(path, flag="n")
        return store, None


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithDBMStore(TestGroupWithDBMStore, TestGroupV3):
    @staticmethod
    def create_store():
        path = mktemp(suffix=".anydbm")
        atexit.register(atexit_rmglob, path + "*")
        store = DBMStoreV3(path, flag="n")
        return store, None


class TestGroupWithDBMStoreBerkeleyDB(TestGroup):
    @staticmethod
    def create_store():
        bsddb3 = pytest.importorskip("bsddb3")
        path = mktemp(suffix=".dbm")
        atexit.register(os.remove, path)
        store = DBMStore(path, flag="n", open=bsddb3.btopen)
        return store, None


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithDBMStoreBerkeleyDB(TestGroupWithDBMStoreBerkeleyDB, TestGroupV3):
    @staticmethod
    def create_store():
        bsddb3 = pytest.importorskip("bsddb3")
        path = mktemp(suffix=".dbm")
        atexit.register(os.remove, path)
        store = DBMStoreV3(path, flag="n", open=bsddb3.btopen)
        return store, None


class TestGroupWithLMDBStore(TestGroup):
    @staticmethod
    def create_store():
        pytest.importorskip("lmdb")
        path = mktemp(suffix=".lmdb")
        atexit.register(atexit_rmtree, path)
        store = LMDBStore(path)
        return store, None


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithLMDBStore(TestGroupWithLMDBStore, TestGroupV3):
    @staticmethod
    def create_store():
        pytest.importorskip("lmdb")
        path = mktemp(suffix=".lmdb")
        atexit.register(atexit_rmtree, path)
        store = LMDBStoreV3(path)
        return store, None


class TestGroupWithSQLiteStore(TestGroup):
    def create_store(self):
        pytest.importorskip("sqlite3")
        path = mktemp(suffix=".db")
        atexit.register(atexit_rmtree, path)
        store = SQLiteStore(path)
        return store, None


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithSQLiteStore(TestGroupWithSQLiteStore, TestGroupV3):
    def create_store(self):
        pytest.importorskip("sqlite3")
        path = mktemp(suffix=".db")
        atexit.register(atexit_rmtree, path)
        store = SQLiteStoreV3(path)
        return store, None


class TestGroupWithChunkStore(TestGroup):
    @staticmethod
    def create_store():
        return KVStore(dict()), KVStore(dict())

    def test_chunk_store(self):
        # setup
        store, chunk_store = self.create_store()
        g = self.create_group(store, chunk_store=chunk_store)

        # check attributes
        assert store is g.store
        assert chunk_store is g.chunk_store

        # create array
        a = g.zeros("foo", shape=100, chunks=10)
        assert store is a.store
        assert chunk_store is a.chunk_store
        a[:] = np.arange(100)
        assert_array_equal(np.arange(100), a[:])

        # check store keys
        expect = sorted([group_meta_key, "foo/" + array_meta_key])
        actual = sorted(store.keys())
        assert expect == actual
        expect = ["foo/" + str(i) for i in range(10)]
        actual = sorted(chunk_store.keys())
        assert expect == actual


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithChunkStore(TestGroupWithChunkStore, TestGroupV3):
    @staticmethod
    def create_store():
        return KVStoreV3(dict()), KVStoreV3(dict())

    def test_chunk_store(self):
        # setup
        store, chunk_store = self.create_store()
        path = "group1"
        g = self.create_group(store, path=path, chunk_store=chunk_store)

        # check attributes
        assert store is g.store
        assert chunk_store is g.chunk_store

        # create array
        a = g.zeros("foo", shape=100, chunks=10)
        assert store is a.store
        assert chunk_store is a.chunk_store
        a[:] = np.arange(100)
        assert_array_equal(np.arange(100), a[:])

        # check store keys
        group_key = meta_root + path + ".group.json"
        array_key = meta_root + path + "/foo" + ".array.json"
        expect = sorted([group_key, array_key, "zarr.json"])
        actual = sorted(store.keys())
        assert expect == actual
        expect = [data_root + path + "/foo/c" + str(i) for i in range(10)]
        expect += ["zarr.json"]
        actual = sorted(chunk_store.keys())
        assert expect == actual


class TestGroupWithStoreCache(TestGroup):
    @staticmethod
    def create_store():
        store = LRUStoreCache(dict(), max_size=None)
        return store, None


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestGroupV3WithStoreCache(TestGroupWithStoreCache, TestGroupV3):
    @staticmethod
    def create_store():
        store = LRUStoreCacheV3(dict(), max_size=None)
        return store, None


@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_group(zarr_version):
    # test the group() convenience function

    # basic usage
    if zarr_version == 2:
        g = group()
        assert "" == g.path
        assert "/" == g.name
    else:
        g = group(path="group1", zarr_version=zarr_version)
        assert "group1" == g.path
        assert "/group1" == g.name
    assert isinstance(g, Group)

    # usage with custom store
    if zarr_version == 2:
        store = KVStore(dict())
        path = None
    else:
        store = KVStoreV3(dict())
        path = "foo"
    g = group(store=store, path=path)
    assert isinstance(g, Group)
    assert store is g.store

    # overwrite behaviour
    if zarr_version == 2:
        store = KVStore(dict())
        path = None
    else:
        store = KVStoreV3(dict())
        path = "foo"
    init_array(store, path=path, shape=100, chunks=10)
    with pytest.raises(ValueError):
        group(store, path=path)
    g = group(store, path=path, overwrite=True)
    assert isinstance(g, Group)
    assert store is g.store


@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_group_writeable_mode(zarr_version, tmp_path):
    # Regression test for https://github.com/zarr-developers/zarr-python/issues/1353
    import fsspec

    store = fsspec.get_mapper(str(tmp_path))
    zg = group(store=store)
    assert zg.store.map == store


@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(zarr_version):
    # test the open_group() convenience function

    store = "data/group.zarr"

    expected_store_type = DirectoryStore if zarr_version == 2 else DirectoryStoreV3

    # mode == 'w'
    path = None if zarr_version == 2 else "group1"
    g = open_group(store, path=path, mode="w", zarr_version=zarr_version)
    assert isinstance(g, Group)
    assert isinstance(g.store, expected_store_type)
    assert 0 == len(g)
    g.create_groups("foo", "bar")
    assert 2 == len(g)

    # mode in 'r', 'r+'
    open_array("data/array.zarr", shape=100, chunks=10, mode="w")
    for mode in "r", "r+":
        with pytest.raises(ValueError):
            open_group("doesnotexist", mode=mode)
        with pytest.raises(ValueError):
            open_group("data/array.zarr", mode=mode)
    g = open_group(store, mode="r")
    assert isinstance(g, Group)
    assert 2 == len(g)
    with pytest.raises(PermissionError):
        g.create_group("baz")
    g = open_group(store, mode="r+")
    assert isinstance(g, Group)
    assert 2 == len(g)
    g.create_groups("baz", "quux")
    assert 4 == len(g)

    # mode == 'a'
    shutil.rmtree(store)
    g = open_group(store, path=path, mode="a", zarr_version=zarr_version)
    assert isinstance(g, Group)
    assert isinstance(g.store, expected_store_type)
    assert 0 == len(g)
    g.create_groups("foo", "bar")
    assert 2 == len(g)
    if zarr_version == 2:
        with pytest.raises(ValueError):
            open_group("data/array.zarr", mode="a", zarr_version=zarr_version)
    else:
        # TODO, root: should this raise an error?
        open_group("data/array.zarr", mode="a", zarr_version=zarr_version)

    # mode in 'w-', 'x'
    for mode in "w-", "x":
        shutil.rmtree(store)
        g = open_group(store, path=path, mode=mode, zarr_version=zarr_version)
        assert isinstance(g, Group)
        assert isinstance(g.store, expected_store_type)
        assert 0 == len(g)
        g.create_groups("foo", "bar")
        assert 2 == len(g)
        with pytest.raises(ValueError):
            open_group(store, path=path, mode=mode, zarr_version=zarr_version)
        if zarr_version == 2:
            with pytest.raises(ValueError):
                open_group("data/array.zarr", mode=mode)

    # open with path
    g = open_group(store, path="foo/bar", zarr_version=zarr_version)
    assert isinstance(g, Group)
    assert "foo/bar" == g.path


@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_group_completions(zarr_version):
    path = None if zarr_version == 2 else "group1"
    g = group(path=path, zarr_version=zarr_version)
    d = dir(g)
    assert "foo" not in d
    assert "bar" not in d
    assert "baz" not in d
    assert "qux" not in d
    assert "xxx" not in d
    assert "yyy" not in d
    assert "zzz" not in d
    assert "123" not in d
    assert "456" not in d
    g.create_groups("foo", "bar", "baz/qux", "123")
    g.zeros("xxx", shape=100)
    g.zeros("yyy", shape=100)
    g.zeros("zzz", shape=100)
    g.zeros("456", shape=100)
    d = dir(g)
    assert "foo" in d
    assert "bar" in d
    assert "baz" in d
    assert "qux" not in d
    assert "xxx" in d
    assert "yyy" in d
    assert "zzz" in d
    assert "123" not in d  # not valid identifier
    assert "456" not in d  # not valid identifier


@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_group_key_completions(zarr_version):
    path = None if zarr_version == 2 else "group1"
    g = group(path=path, zarr_version=zarr_version)
    d = dir(g)
    # noinspection PyProtectedMember
    k = g._ipython_key_completions_()

    # none of these names should be an attribute
    assert "foo" not in d
    assert "bar" not in d
    assert "baz" not in d
    assert "qux" not in d
    assert "xxx" not in d
    assert "yyy" not in d
    assert "zzz" not in d
    assert "123" not in d
    assert "456" not in d
    assert "asdf;" not in d

    # none of these names should be an item
    assert "foo" not in k
    assert "bar" not in k
    assert "baz" not in k
    assert "qux" not in k
    assert "xxx" not in k
    assert "yyy" not in k
    assert "zzz" not in k
    assert "123" not in k
    assert "456" not in k
    assert "asdf;" not in k

    g.create_groups("foo", "bar", "baz/qux", "123")
    g.zeros("xxx", shape=100)
    g.zeros("yyy", shape=100)
    g.zeros("zzz", shape=100)
    g.zeros("456", shape=100)
    if zarr_version == 2:
        g.zeros("asdf;", shape=100)
    else:
        # cannot have ; in key name for v3
        with pytest.raises(ValueError):
            g.zeros("asdf;", shape=100)

    d = dir(g)
    # noinspection PyProtectedMember
    k = g._ipython_key_completions_()

    assert "foo" in d
    assert "bar" in d
    assert "baz" in d
    assert "qux" not in d
    assert "xxx" in d
    assert "yyy" in d
    assert "zzz" in d
    assert "123" not in d  # not valid identifier
    assert "456" not in d  # not valid identifier
    if zarr_version == 2:
        assert "asdf;" not in d  # not valid identifier

    assert "foo" in k
    assert "bar" in k
    assert "baz" in k
    assert "qux" not in k
    assert "xxx" in k
    assert "yyy" in k
    assert "zzz" in k
    assert "123" in k
    assert "456" in k
    if zarr_version == 2:
        assert "asdf;" in k


def _check_tree(g, expect_bytes, expect_text):
    assert expect_bytes == bytes(g.tree())
    assert expect_text == str(g.tree())
    expect_repr = expect_text
    assert expect_repr == repr(g.tree())
    if ipytree:
        # noinspection PyProtectedMember
        widget = g.tree()._repr_mimebundle_()
        isinstance(widget, ipytree.Tree)


@pytest.mark.parametrize("zarr_version", _VERSIONS)
@pytest.mark.parametrize("at_root", [False, True])
def test_tree(zarr_version, at_root):
    # setup
    path = None if at_root else "group1"
    g1 = group(path=path, zarr_version=zarr_version)
    g2 = g1.create_group("foo")
    g3 = g1.create_group("bar")
    g3.create_group("baz")
    g5 = g3.create_group("quux")
    g5.create_dataset("baz", shape=100, chunks=10)

    tree_path = "/" if at_root else path
    # test root group
    if zarr_version == 2:
        expect_bytes = textwrap.dedent(
            f"""\
        {tree_path}
         +-- bar
         |   +-- baz
         |   +-- quux
         |       +-- baz (100,) float64
         +-- foo"""
        ).encode()
        expect_text = textwrap.dedent(
            f"""\
        {tree_path}
         ├── bar
         │   ├── baz
         │   └── quux
         │       └── baz (100,) float64
         └── foo"""
        )
    else:
        # Almost the same as for v2, but has a path name and the
        # subgroups are not necessarily sorted alphabetically.
        expect_bytes = textwrap.dedent(
            f"""\
        {tree_path}
         +-- foo
         +-- bar
             +-- baz
             +-- quux
                 +-- baz (100,) float64"""
        ).encode()
        expect_text = textwrap.dedent(
            f"""\
        {tree_path}
         ├── foo
         └── bar
             ├── baz
             └── quux
                 └── baz (100,) float64"""
        )
    _check_tree(g1, expect_bytes, expect_text)

    # test different group
    expect_bytes = textwrap.dedent(
        """\
    foo"""
    ).encode()
    expect_text = textwrap.dedent(
        """\
    foo"""
    )
    _check_tree(g2, expect_bytes, expect_text)

    # test different group
    expect_bytes = textwrap.dedent(
        """\
    bar
     +-- baz
     +-- quux
         +-- baz (100,) float64"""
    ).encode()
    expect_text = textwrap.dedent(
        """\
    bar
     ├── baz
     └── quux
         └── baz (100,) float64"""
    )
    _check_tree(g3, expect_bytes, expect_text)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_group_mismatched_store_versions():
    store_v3 = KVStoreV3(dict())
    store_v2 = KVStore(dict())

    # separate chunk store
    chunk_store_v2 = KVStore(dict())
    chunk_store_v3 = KVStoreV3(dict())

    init_group(store_v2, path="group1", chunk_store=chunk_store_v2)
    init_group(store_v3, path="group1", chunk_store=chunk_store_v3)

    g1_v3 = Group(store_v3, path="group1", read_only=True, chunk_store=chunk_store_v3)
    assert isinstance(g1_v3._store, KVStoreV3)
    g1_v2 = Group(store_v2, path="group1", read_only=True, chunk_store=chunk_store_v2)
    assert isinstance(g1_v2._store, KVStore)

    # store and chunk_store must have the same zarr protocol version
    with pytest.raises(ValueError):
        Group(store_v3, path="group1", read_only=False, chunk_store=chunk_store_v2)
    with pytest.raises(ValueError):
        Group(store_v2, path="group1", read_only=False, chunk_store=chunk_store_v3)
    with pytest.raises(ValueError):
        open_group(store_v2, path="group1", chunk_store=chunk_store_v3)
    with pytest.raises(ValueError):
        open_group(store_v3, path="group1", chunk_store=chunk_store_v2)

    # raises Value if read_only and path is not a pre-existing group
    with pytest.raises(ValueError):
        Group(store_v3, path="group2", read_only=True, chunk_store=chunk_store_v3)
    with pytest.raises(ValueError):
        Group(store_v3, path="group2", read_only=True, chunk_store=chunk_store_v3)


@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group_from_paths(zarr_version):
    """Verify zarr_version is applied to both the store and chunk_store."""
    store = tempfile.mkdtemp()
    chunk_store = tempfile.mkdtemp()
    atexit.register(atexit_rmtree, store)
    atexit.register(atexit_rmtree, chunk_store)
    path = "g1"
    g = open_group(store, path=path, chunk_store=chunk_store, zarr_version=zarr_version)
    assert g._store._store_version == g._chunk_store._store_version == zarr_version