Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add NOT operator to VectorStore portable filter expressions

Browse Source 
- Add NOT expression type to the portable Filter.Expression model.
 - Add NOT to the Antlr grammar and implement the related parser listener method to generate Filter NOT expressions.
 - Add NOT support to the filter programming DSL.
 - Implement FilterHelper.negation for logically transform any boolean expression with NOT statements into
   semantically equivalent one with NOT applied to the leaf expressions.
 - Add tests for paresers, converters and vectorsores ITs.
 - Move the filter IN/NIN expansion logic to the FilterHelper
 - Factor out the filter IN/NIN boolean expression expansion logic out of Weaviate up to the FilterHelper.
 - add in/nin expantion FilterHelper tests
This commit is contained in:
Christian Tzolov
2023-11-23 21:29:48 +01:00
committed by Mark Pollack
parent e3e5070256
commit ac9ae589f4
25 changed files with 731 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
vector-stores/spring-ai-azure/src/test/java/org/springframework/ai/vectorstore/azure/AzureVectorStoreIT.java
View File

@@ -166,11 +166,20 @@ public class AzureVectorStoreIT {
results = vectorStore.similaritySearch(SearchRequest.query("The World")
.withTopK(5)
.withSimilarityThresholdAll()
.withFilterExpression("country nin ['BG']"));
.withFilterExpression("country not in ['BG']"));
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId());
results = vectorStore.similaritySearch(SearchRequest.query("The World")
.withTopK(5)
.withSimilarityThresholdAll()
.withFilterExpression("NOT(country not in ['BG'])"));
assertThat(results).hasSize(2);
assertThat(results.get(0).getId()).isIn(bgDocument.getId(), bgDocument2.getId());
assertThat(results.get(1).getId()).isIn(bgDocument.getId(), bgDocument2.getId());
// List<Document> results =
// vectorStore.similaritySearch(SearchRequest.query("The World")
// .withTopK(5)

5
vector-stores/spring-ai-chroma/src/test/java/org/springframework/experimental/ai/vectorstore/ChromaVectorStoreIT.java
View File

@@ -122,6 +122,11 @@ public class ChromaVectorStoreIT {
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId());
results = vectorStore.similaritySearch(
request.withSimilarityThresholdAll().withFilterExpression("NOT(country == 'Netherland')"));
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(bgDocument.getId());
// Remove all documents from the store
vectorStore.delete(List.of(bgDocument, nlDocument).stream().map(doc -> doc.getId()).toList());
});

10
vector-stores/spring-ai-milvus-store/src/test/java/org/springframework/ai/vectorstore/MilvusVectorStoreIT.java
View File

@@ -188,6 +188,16 @@ public class MilvusVectorStoreIT {
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(bgDocument.getId());
results = vectorStore.similaritySearch(SearchRequest.query("The World")
.withTopK(5)
.withSimilarityThresholdAll()
.withFilterExpression("NOT(country == 'BG' && year == 2020)"));
assertThat(results).hasSize(2);
assertThat(results.get(0).getId()).isIn(nlDocument.getId(), bgDocument2.getId());
assertThat(results.get(1).getId()).isIn(nlDocument.getId(), bgDocument2.getId());
});
}

6
vector-stores/spring-ai-pgvector-store/src/test/java/org/springframework/ai/vectorstore/PgVectorStoreIT.java
View File

@@ -182,6 +182,12 @@ public class PgVectorStoreIT {
assertThat(results.get(0).getId()).isIn(bgDocument.getId(), nlDocument.getId());
assertThat(results.get(1).getId()).isIn(bgDocument.getId(), nlDocument.getId());
results = vectorStore.similaritySearch(searchRequest
.withFilterExpression("NOT((country == 'BG' && year == 2020) || (country == 'NL'))"));
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(bgDocument2.getId());
results = vectorStore.similaritySearch(SearchRequest.query("The World")
.withTopK(5)
.withSimilarityThresholdAll()

7
vector-stores/spring-ai-pinecone/src/test/java/org/springframework/ai/vectorstore/PineconeVectorStoreIT.java
View File

@@ -156,6 +156,13 @@ public class PineconeVectorStoreIT {
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId());
results = vectorStore.similaritySearch(searchRequest.withTopK(5)
.withSimilarityThresholdAll()
.withFilterExpression("NOT(country == 'Netherland')"));
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(bgDocument.getId());
// Remove all documents from the store
vectorStore.delete(List.of(bgDocument, nlDocument).stream().map(doc -> doc.getId()).toList());

51
vector-stores/spring-ai-weaviate/src/main/java/org/springframework/ai/vectorstore/WeaviateFilterExpressionConverter.java
View File

@@ -16,7 +16,6 @@
package org.springframework.ai.vectorstore;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -27,6 +26,7 @@ import org.springframework.ai.vectorstore.filter.Filter.Expression;
import org.springframework.ai.vectorstore.filter.Filter.ExpressionType;
import org.springframework.ai.vectorstore.filter.Filter.Group;
import org.springframework.ai.vectorstore.filter.Filter.Key;
import org.springframework.ai.vectorstore.filter.FilterHelper;
import org.springframework.ai.vectorstore.filter.converter.AbstractFilterExpressionConverter;
import org.springframework.util.Assert;
@@ -62,10 +62,10 @@ public class WeaviateFilterExpressionConverter extends AbstractFilterExpressionC
protected void doExpression(Expression exp, StringBuilder context) {
if (exp.type() == ExpressionType.IN) {
rewriteInNinExpressions(Filter.ExpressionType.OR, Filter.ExpressionType.EQ, exp, context);
FilterHelper.expandIn(exp, context, this);
}
else if (exp.type() == ExpressionType.NIN) {
rewriteInNinExpressions(Filter.ExpressionType.AND, Filter.ExpressionType.NE, exp, context);
FilterHelper.expandNin(exp, context, this);
}
else if (exp.type() == ExpressionType.AND || exp.type() == ExpressionType.OR) {
context.append(getOperationSymbol(exp));
@@ -82,51 +82,6 @@ public class WeaviateFilterExpressionConverter extends AbstractFilterExpressionC
}
}
/**
* Recursively aggregates a list of expression into a binary tree with 'aggregateType'
* join nodes.
* @param aggregateType type all tree splits.
* @param expressions list of expressions to aggregate.
* @return Returns a binary tree expression.
*/
private Filter.Expression aggregate(Filter.ExpressionType aggregateType, List<Filter.Expression> expressions) {
if (expressions.size() == 1) {
return expressions.get(0);
}
return new Filter.Expression(aggregateType, expressions.get(0),
aggregate(aggregateType, expressions.subList(1, expressions.size())));
}
private void rewriteInNinExpressions(Filter.ExpressionType outerExpressionType,
Filter.ExpressionType innerExpressionType, Expression exp, StringBuilder context) {
if (exp.right() instanceof Filter.Value value) {
if (value.value() instanceof List list) {
// 1. foo IN ["bar1", "bar2", "bar3"] is equivalent to foo == "bar1" ||
// foo == "bar2" || foo == "bar3"
// or equivalent to OR(foo == "bar1" OR( foo == "bar2" OR(foo == "bar3")))
// 2. foo IN ["bar1", "bar2", "bar3"] is equivalent to foo != "bar1" &&
// foo != "bar2" && foo != "bar3"
// or equivalent to AND(foo != "bar1" AND( foo != "bar2" OR(foo !=
// "bar3")))
List<Filter.Expression> eqExprs = new ArrayList<>();
for (Object o : list) {
eqExprs.add(new Filter.Expression(innerExpressionType, exp.left(), new Filter.Value(o)));
}
this.doExpression(aggregate(outerExpressionType, eqExprs), context);
}
else {
// 1. foo IN ["bar"] is equivalent to foo == "BAR"
// 2. foo NIN ["bar"] is equivalent to foo != "BAR"
this.doExpression(new Filter.Expression(innerExpressionType, exp.left(), exp.right()), context);
}
}
else {
throw new IllegalStateException(
"Filter IN right expression should be of Filter.Value type but was " + exp.right().getClass());
}
}
private String getOperationSymbol(Expression exp) {
switch (exp.type()) {
case AND:

1
vector-stores/spring-ai-weaviate/src/test/java/org/springframework/ai/vectorstore/WeaviateFilterExpressionConverterTests.java
View File

@@ -20,7 +20,6 @@ import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.ai.vectorstore.filter.Filter;
import org.springframework.ai.vectorstore.filter.Filter.Expression;
import org.springframework.ai.vectorstore.filter.Filter.Group;
import org.springframework.ai.vectorstore.filter.Filter.Key;

8
vector-stores/spring-ai-weaviate/src/test/java/org/springframework/ai/vectorstore/WeaviateVectorStoreIT.java
View File

@@ -152,6 +152,14 @@ public class WeaviateVectorStoreIT {
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(bgDocument.getId());
results = vectorStore.similaritySearch(SearchRequest.query("The World")
.withTopK(5)
.withSimilarityThresholdAll()
.withFilterExpression("NOT((country == 'BG' && year == 2020) || (country == 'NL'))"));
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(bgDocument2.getId());
vectorStore.delete(List.of(bgDocument.getId(), nlDocument.getId(), bgDocument2.getId()));
});
}