implement any and all metadata filters for weaviate vector store

llama-index-core/llama_index/core/vector_stores/simple.py

@@ -46,7 +46,7 @@ def _build_metadata_filter_fn(
     metadata_filters: Optional[MetadataFilters] = None,
 ) -> Callable[[str], bool]:
     """Build metadata filter function."""
-    filter_list = metadata_filters.legacy_filters() if metadata_filters else []
+    filter_list = metadata_filters.filters if metadata_filters else []
     if not filter_list:
         return lambda _: True
 
@@ -62,8 +62,17 @@ def filter_fn(node_id: str) -> bool:
             if metadata_value is None:
                 filter_matches = False
             elif isinstance(metadata_value, list):
-                if filter_.value not in metadata_value:
-                    filter_matches = False
+                match filter_.operator:
+                    case "any":
+                        if not any(value in metadata_value for value in filter_.value):
+                            filter_matches = False
+                    case "all":
+                        for value in filter_.value:
+                            if value not in metadata_value:
+                                filter_matches = False
+                    case _:
+                        if filter_.value not in metadata_value:
+                            filter_matches = False
             elif isinstance(metadata_value, (int, float, str, bool)):
                 if metadata_value != filter_.value:
                     filter_matches = False

llama-index-core/llama_index/core/vector_stores/types.py

@@ -1,5 +1,4 @@
 """Vector store index types."""
-
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from enum import Enum
@@ -69,10 +68,11 @@ class FilterOperator(str, Enum):
     NE = "!="  # not equal to (string, int, float)
     GTE = ">="  # greater than or equal to (int, float)
     LTE = "<="  # less than or equal to (int, float)
-    IN = "in"  # metadata in value array (string or number)
-    NIN = "nin"  # metadata not in value array (string or number)
+    IN = "in"  # In array (string or number)
+    NIN = "nin"  # Not in array (string or number)
+    ANY = "any"  # Contains any (array of strings)
+    ALL = "all"  # Contains all (array of strings)
     TEXT_MATCH = "text_match"  # full text match (allows you to search for a specific substring, token or phrase within the text field)
-    CONTAINS = "contains"  # metadata array contains value (string or number)
 
 
 class FilterCondition(str, Enum):
@@ -93,12 +93,7 @@ class MetadataFilter(BaseModel):
     """
 
     key: str
-    value: Union[
-        StrictInt,
-        StrictFloat,
-        StrictStr,
-        List[Union[StrictInt, StrictFloat, StrictStr]],
-    ]
+    value: Union[StrictInt, StrictFloat, StrictStr, List[StrictStr]]
     operator: FilterOperator = FilterOperator.EQ
 
     @classmethod

llama-index-core/tests/vector_stores/test_simple.py

@@ -8,6 +8,8 @@
     MetadataFilters,
     VectorStoreQuery,
     FilterCondition,
+    MetadataFilter,
+    FilterOperator,
 )
 
 _NODE_ID_WEIGHT_1_RANK_A = "AF3BE6C4-5F43-4D74-B075-6B0E07900DE8"
@@ -22,21 +24,21 @@ def _node_embeddings_for_test() -> List[TextNode]:
             id_=_NODE_ID_WEIGHT_1_RANK_A,
             embedding=[1.0, 0.0],
             relationships={NodeRelationship.SOURCE: RelatedNodeInfo(node_id="test-0")},
-            metadata={"weight": 1.0, "rank": "a"},
+            metadata={"weight": 1.0, "rank": "a", "quality": ["medium", "high"]},
         ),
         TextNode(
             text="lorem ipsum",
             id_=_NODE_ID_WEIGHT_2_RANK_C,
             embedding=[0.0, 1.0],
             relationships={NodeRelationship.SOURCE: RelatedNodeInfo(node_id="test-1")},
-            metadata={"weight": 2.0, "rank": "c"},
+            metadata={"weight": 2.0, "rank": "c", "quality": ["medium"]},
         ),
         TextNode(
             text="lorem ipsum",
             id_=_NODE_ID_WEIGHT_3_RANK_C,
             embedding=[1.0, 1.0],
             relationships={NodeRelationship.SOURCE: RelatedNodeInfo(node_id="test-2")},
-            metadata={"weight": 3.0, "rank": "c"},
+            metadata={"weight": 3.0, "rank": "c", "quality": ["low", "medium", "high"]},
         ),
     ]
 
@@ -181,3 +183,39 @@ def test_query_with_filters_with_filter_condition(self) -> None:
         )
         result = simple_vector_store.query(query)
         self.assertEqual(len(result.ids), 0)
+
+    def test_query_with_any_filter_returns_matches(self) -> None:
+        simple_vector_store = SimpleVectorStore()
+        simple_vector_store.add(_node_embeddings_for_test())
+
+        filters = MetadataFilters(
+            filters=[
+                MetadataFilter(
+                    key="quality", operator=FilterOperator.ANY, value=["high", "low"]
+                )
+            ]
+        )
+        query = VectorStoreQuery(
+            query_embedding=[1.0, 1.0], filters=filters, similarity_top_k=3
+        )
+        result = simple_vector_store.query(query)
+        assert result.ids is not None
+        self.assertEqual(len(result.ids), 2)
+
+    def test_query_with_all_filter_returns_matches(self) -> None:
+        simple_vector_store = SimpleVectorStore()
+        simple_vector_store.add(_node_embeddings_for_test())
+
+        filters = MetadataFilters(
+            filters=[
+                MetadataFilter(
+                    key="quality", operator=FilterOperator.ALL, value=["medium", "high"]
+                )
+            ]
+        )
+        query = VectorStoreQuery(
+            query_embedding=[1.0, 1.0], filters=filters, similarity_top_k=3
+        )
+        result = simple_vector_store.query(query)
+        assert result.ids is not None
+        self.assertEqual(len(result.ids), 2)

llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py

@@ -58,6 +58,10 @@ def _transform_weaviate_filter_operator(operator: str) -> str:
         return "GreaterThanEqual"
     elif operator == "<=":
         return "LessThanEqual"
+    elif operator == "all":
+        return "ContainsAll"
+    elif operator == "any":
+        return "ContainsAny"
     else:
         raise ValueError(f"Filter operator {operator} not supported")
 
@@ -77,6 +81,8 @@ def _to_weaviate_filter(standard_filters: MetadataFilters) -> Dict[str, Any]:
             elif isinstance(filter.value, str) and filter.value.isnumeric():
                 filter.value = float(filter.value)
                 value_type = "valueNumber"
+            if filter.operator in ["any", "all"]:
+                value_type = "valueTextArray"
             filters_list.append(
                 {
                     "path": filter.key,