feat: Allow None metadata filter by using IS_EMPTY operator (run-llama#15167)

Author: leehuwuj

llama-index-core/llama_index/core/vector_stores/simple.py

@@ -91,12 +91,19 @@ def _process_filter_match(
         filter_matches_list = []
         for filter_ in filter_list:
             filter_matches = True
-
-            filter_matches = _process_filter_match(
-                operator=filter_.operator,
-                value=filter_.value,
-                metadata_value=metadata.get(filter_.key, None),
-            )
+            metadata_value = metadata.get(filter_.key, None)
+            if filter_.operator == FilterOperator.IS_EMPTY:
+                filter_matches = (
+                    metadata_value is None
+                    or metadata_value == ""
+                    or metadata_value == []
+                )
+            else:
+                filter_matches = _process_filter_match(
+                    operator=filter_.operator,
+                    value=filter_.value,
+                    metadata_value=metadata_value,
+                )
 
             filter_matches_list.append(filter_matches)
 

llama-index-core/llama_index/core/vector_stores/types.py

@@ -74,6 +74,7 @@ class FilterOperator(str, Enum):
     ALL = "all"  # Contains all (array of strings)
     TEXT_MATCH = "text_match"  # full text match (allows you to search for a specific substring, token or phrase within the text field)
     CONTAINS = "contains"  # metadata array contains value (string or number)
+    IS_EMPTY = "is_empty"  # the field is not exist or empty (null or empty array)
 
 
 class FilterCondition(str, Enum):
@@ -94,13 +95,15 @@ class MetadataFilter(BaseModel):
     """
 
     key: str
-    value: Union[
-        StrictInt,
-        StrictFloat,
-        StrictStr,
-        List[StrictStr],
-        List[StrictFloat],
-        List[StrictInt],
+    value: Optional[
+        Union[
+            StrictInt,
+            StrictFloat,
+            StrictStr,
+            List[StrictStr],
+            List[StrictFloat],
+            List[StrictInt],
+        ]
     ]
     operator: FilterOperator = FilterOperator.EQ
 

llama-index-core/tests/vector_stores/test_simple.py

@@ -400,6 +400,23 @@ def test_query_with_all_filter_returns_matches(self) -> None:
         assert result.ids is not None
         self.assertEqual(len(result.ids), 2)
 
+    def test_query_with_is_empty_filter_returns_matches(self) -> None:
+        simple_vector_store = SimpleVectorStore()
+        simple_vector_store.add(_node_embeddings_for_test())
+
+        filters = MetadataFilters(
+            filters=[
+                MetadataFilter(
+                    key="not_existed_key", operator=FilterOperator.IS_EMPTY, value=None
+                )
+            ]
+        )
+        query = VectorStoreQuery(
+            query_embedding=[1.0, 1.0], filters=filters, similarity_top_k=3
+        )
+        result = simple_vector_store.query(query)
+        self.assertEqual(len(result.ids), len(_node_embeddings_for_test()))
+
     def test_clear(self) -> None:
         simple_vector_store = SimpleVectorStore()
         simple_vector_store.add(_node_embeddings_for_test())

llama-index-integrations/vector_stores/llama-index-vector-stores-qdrant/llama_index/vector_stores/qdrant/base.py

@@ -44,8 +44,10 @@
     MatchText,
     MatchValue,
     Payload,
+    PayloadField,
     Range,
     HasIdCondition,
+    IsEmptyCondition,
 )
 
 logger = logging.getLogger(__name__)
@@ -1095,6 +1097,12 @@ def _build_subfilter(self, filters: MetadataFilters) -> Filter:
                         match=MatchExcept(**{"except": values}),
                     )
                 )
+            elif subfilter.operator == FilterOperator.IS_EMPTY:
+                # This condition will match all records where the field reports either does not exist, or has null or [] value.
+                # https://qdrant.tech/documentation/concepts/filtering/#is-empty
+                conditions.append(
+                    IsEmptyCondition(is_empty=PayloadField(key=subfilter.key))
+                )
 
         filter = Filter()
         if filters.condition == FilterCondition.AND: