Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add filter-based deletion to Neo4j and OpenSearch vector stores

Browse Source 
Add string-based filter deletion alongside the Filter.Expression-based deletion
for Neo4j and OpenSearch vector stores, providing consistent deletion capabilities
with other vector store implementations.

Key changes:
- Add delete(Filter.Expression) implementation for Neo4j store using Cypher queries
- Add delete(Filter.Expression) implementation for OpenSearch store using query_string
- Leverage existing filter expression converters for both stores
- Use Neo4j's transaction batching for efficient large-scale deletions
- Use OpenSearch's delete_by_query API for metadata-based deletion
- Add comprehensive integration tests for both stores covering:
  * Simple equality filters
  * String-based filter expressions
  * Complex filter expressions with multiple conditions

This maintains consistency with other vector store implementations while utilizing
store-specific features for efficient metadata-based deletion.

Signed-off-by: Soby Chacko <soby.chacko@broadcom.com>
This commit is contained in:
Soby Chacko
2025-01-27 16:04:04 -05:00
committed by Mark Pollack
parent ef0bade409
commit 07b905896a
4 changed files with 268 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
vector-stores/spring-ai-neo4j-store/src/main/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStore.java
View File

@@ -21,6 +21,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.logging.LogFactory;
import org.neo4j.cypherdsl.support.schema_name.SchemaNames;
import org.neo4j.driver.Driver;
import org.neo4j.driver.SessionConfig;
@@ -28,18 +29,18 @@ import org.neo4j.driver.Values;
import org.springframework.ai.document.Document;
import org.springframework.ai.document.DocumentMetadata;
import org.springframework.ai.embedding.BatchingStrategy;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.embedding.EmbeddingOptionsBuilder;
import org.springframework.ai.embedding.TokenCountBatchingStrategy;
import org.springframework.ai.observation.conventions.VectorStoreProvider;
import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric;
import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.filter.Filter;
import org.springframework.ai.vectorstore.neo4j.filter.Neo4jVectorFilterExpressionConverter;
import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.core.log.LogAccessor;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
@@ -133,6 +134,8 @@ import org.springframework.util.StringUtils;
*/
public class Neo4jVectorStore extends AbstractObservationVectorStore implements InitializingBean {
private static final LogAccessor logger = new LogAccessor(LogFactory.getLog(Neo4jVectorStore.class));
public static final int DEFAULT_EMBEDDING_DIMENSION = 1536;
public static final int DEFAULT_TRANSACTION_SIZE = 10_000;
@@ -235,6 +238,29 @@ public class Neo4jVectorStore extends AbstractObservationVectorStore implements
}
}
@Override
protected void doDelete(Filter.Expression filterExpression) {
Assert.notNull(filterExpression, "Filter expression must not be null");
try (var session = this.driver.session(this.sessionConfig)) {
String whereClause = this.filterExpressionConverter.convertExpression(filterExpression);
// Create Cypher query with transaction batching
String cypher = """
MATCH (node:%s) WHERE %s
CALL { WITH node DETACH DELETE node } IN TRANSACTIONS OF $transactionSize ROWS
""".formatted(this.label, whereClause);
var summary = session.run(cypher, Map.of("transactionSize", DEFAULT_TRANSACTION_SIZE)).consume();
logger.debug("Deleted " + summary.counters().nodesDeleted() + " nodes matching filter expression");
}
catch (Exception e) {
logger.error(e, "Failed to delete nodes by filter: " + e.getMessage());
throw new IllegalStateException("Failed to delete nodes by filter", e);
}
}
@Override
public List<Document> doSimilaritySearch(SearchRequest request) {
Assert.isTrue(request.getTopK() > 0, "The number of documents to returned must be greater than zero");

98
vector-stores/spring-ai-neo4j-store/src/test/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStoreIT.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2024 the original author or authors.
* Copyright 2023-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import org.junit.Assert;
import org.junit.jupiter.api.BeforeEach;
@@ -39,6 +40,7 @@ import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.filter.Filter;
import org.springframework.ai.vectorstore.filter.FilterExpressionTextParser;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -53,6 +55,7 @@ import static org.assertj.core.api.Assertions.assertThat;
* @author Michael Simons
* @author Christian Tzolov
* @author Thomas Vitale
* @author Soby Chacko
*/
@Testcontainers
@EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+")
@@ -301,6 +304,99 @@ class Neo4jVectorStoreIT {
.isTrue());
}
@Test
void deleteByFilter() {
this.contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2020));
var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "NL", "year", 2021));
var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2023));
vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
SearchRequest searchRequest = SearchRequest.builder()
.query("The World")
.topK(5)
.similarityThresholdAll()
.build();
List<Document> results = vectorStore.similaritySearch(searchRequest);
assertThat(results).hasSize(3);
Filter.Expression filterExpression = new Filter.Expression(Filter.ExpressionType.EQ,
new Filter.Key("country"), new Filter.Value("BG"));
vectorStore.delete(filterExpression);
results = vectorStore.similaritySearch(searchRequest);
assertThat(results).hasSize(1);
assertThat(results.get(0).getMetadata()).containsEntry("country", "NL");
});
}
@Test
void deleteWithStringFilterExpression() {
this.contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2020));
var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "NL", "year", 2021));
var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2023));
vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
var searchRequest = SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build();
List<Document> results = vectorStore.similaritySearch(searchRequest);
assertThat(results).hasSize(3);
vectorStore.delete("country == 'BG'");
results = vectorStore.similaritySearch(searchRequest);
assertThat(results).hasSize(1);
assertThat(results.get(0).getMetadata()).containsEntry("country", "NL");
});
}
@Test
void deleteWithComplexFilterExpression() {
this.contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
var doc1 = new Document("Content 1", Map.of("type", "A", "priority", 1L));
var doc2 = new Document("Content 2", Map.of("type", "A", "priority", 2L));
var doc3 = new Document("Content 3", Map.of("type", "B", "priority", 1L));
vectorStore.add(List.of(doc1, doc2, doc3));
// Complex filter expression: (type == 'A' AND priority > 1)
Filter.Expression priorityFilter = new Filter.Expression(Filter.ExpressionType.GT,
new Filter.Key("priority"), new Filter.Value(1));
Filter.Expression typeFilter = new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("type"),
new Filter.Value("A"));
Filter.Expression complexFilter = new Filter.Expression(Filter.ExpressionType.AND, typeFilter,
priorityFilter);
vectorStore.delete(complexFilter);
var results = vectorStore
.similaritySearch(SearchRequest.builder().query("Content").topK(5).similarityThresholdAll().build());
assertThat(results).hasSize(2);
assertThat(results.stream().map(doc -> doc.getMetadata().get("type")).collect(Collectors.toList()))
.containsExactlyInAnyOrder("A", "B");
assertThat(results.stream().map(doc -> doc.getMetadata().get("priority")).collect(Collectors.toList()))
.containsExactlyInAnyOrder(1L, 1L);
});
}
@SpringBootConfiguration
@EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class })
public static class TestApplication {

33
vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStore.java
View File

@@ -24,6 +24,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.commons.logging.LogFactory;
import org.opensearch.client.json.JsonData;
import org.opensearch.client.json.JsonpMapper;
import org.opensearch.client.opensearch.OpenSearchClient;
@@ -32,6 +33,8 @@ import org.opensearch.client.opensearch._types.mapping.TypeMapping;
import org.opensearch.client.opensearch._types.query_dsl.Query;
import org.opensearch.client.opensearch.core.BulkRequest;
import org.opensearch.client.opensearch.core.BulkResponse;
import org.opensearch.client.opensearch.core.DeleteByQueryRequest;
import org.opensearch.client.opensearch.core.DeleteByQueryResponse;
import org.opensearch.client.opensearch.core.search.Hit;
import org.opensearch.client.opensearch.indices.CreateIndexRequest;
import org.opensearch.client.opensearch.indices.CreateIndexResponse;
@@ -39,10 +42,8 @@ import org.opensearch.client.transport.endpoints.BooleanResponse;
import org.springframework.ai.document.Document;
import org.springframework.ai.document.DocumentMetadata;
import org.springframework.ai.embedding.BatchingStrategy;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.embedding.EmbeddingOptionsBuilder;
import org.springframework.ai.embedding.TokenCountBatchingStrategy;
import org.springframework.ai.observation.conventions.VectorStoreProvider;
import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric;
import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder;
@@ -52,6 +53,7 @@ import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.core.log.LogAccessor;
import org.springframework.util.Assert;
/**
@@ -139,6 +141,8 @@ import org.springframework.util.Assert;
*/
public class OpenSearchVectorStore extends AbstractObservationVectorStore implements InitializingBean {
private static final LogAccessor logger = new LogAccessor(LogFactory.getLog(OpenSearchVectorStore.class));
public static final String COSINE_SIMILARITY_FUNCTION = "cosinesimil";
public static final String DEFAULT_INDEX_NAME = "spring-ai-document-index";
@@ -230,6 +234,31 @@ public class OpenSearchVectorStore extends AbstractObservationVectorStore implem
}
}
@Override
protected void doDelete(Filter.Expression filterExpression) {
Assert.notNull(filterExpression, "Filter expression must not be null");
try {
String filterStr = this.filterExpressionConverter.convertExpression(filterExpression);
// Create delete by query request
DeleteByQueryRequest request = new DeleteByQueryRequest.Builder().index(this.index)
.query(q -> q.queryString(qs -> qs.query(filterStr)))
.build();
DeleteByQueryResponse response = this.openSearchClient.deleteByQuery(request);
logger.debug("Deleted " + response.deleted() + " documents matching filter expression");
if (!response.failures().isEmpty()) {
throw new IllegalStateException("Failed to delete some documents: " + response.failures());
}
}
catch (Exception e) {
logger.error(e, "Failed to delete documents by filter: " + e.getMessage());
throw new IllegalStateException("Failed to delete documents by filter", e);
}
}
@Override
public List<Document> doSimilaritySearch(SearchRequest searchRequest) {
Assert.notNull(searchRequest, "The search request must not be null.");

113
vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreIT.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2024 the original author or authors.
* Copyright 2023-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,13 +19,16 @@ package org.springframework.ai.vectorstore.opensearch;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.hc.core5.http.HttpHost;
import org.awaitility.Awaitility;
@@ -49,6 +52,7 @@ import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.filter.Filter;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -413,6 +417,113 @@ class OpenSearchVectorStoreIT {
});
}
@Test
void deleteByFilter() {
getContextRunner().run(context -> {
OpenSearchVectorStore vectorStore = context.getBean("vectorStore", OpenSearchVectorStore.class);
var bgDocument = new Document("1", "The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2020, "activationDate", new Date(1000)));
var nlDocument = new Document("2", "The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "NL", "activationDate", new Date(2000)));
var bgDocument2 = new Document("3", "The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2023, "activationDate", new Date(3000)));
vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
Awaitility.await()
.until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("The World").topK(5).build()),
hasSize(3));
Filter.Expression filterExpression = new Filter.Expression(Filter.ExpressionType.EQ,
new Filter.Key("country"), new Filter.Value("BG"));
vectorStore.delete(filterExpression);
Awaitility.await()
.until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("The World").topK(5).build()),
hasSize(1));
List<Document> results = vectorStore
.similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build());
assertThat(results).hasSize(1);
assertThat(results.get(0).getMetadata()).containsEntry("country", "NL");
});
}
@Test
void deleteWithStringFilterExpression() {
getContextRunner().run(context -> {
OpenSearchVectorStore vectorStore = context.getBean("vectorStore", OpenSearchVectorStore.class);
var bgDocument = new Document("1", "The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2020, "activationDate", new Date(1000)));
var nlDocument = new Document("2", "The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "NL", "activationDate", new Date(2000)));
var bgDocument2 = new Document("3", "The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2023, "activationDate", new Date(3000)));
vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
Awaitility.await()
.until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("The World").topK(5).build()),
hasSize(3));
vectorStore.delete("country == 'BG'");
Awaitility.await()
.until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("The World").topK(5).build()),
hasSize(1));
List<Document> results = vectorStore
.similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build());
assertThat(results).hasSize(1);
assertThat(results.get(0).getMetadata()).containsEntry("country", "NL");
});
}
@Test
void deleteWithComplexFilterExpression() {
getContextRunner().run(context -> {
OpenSearchVectorStore vectorStore = context.getBean("vectorStore", OpenSearchVectorStore.class);
var doc1 = new Document("1", "Content 1", Map.of("type", "A", "priority", 1));
var doc2 = new Document("2", "Content 2", Map.of("type", "A", "priority", 2));
var doc3 = new Document("3", "Content 3", Map.of("type", "B", "priority", 1));
vectorStore.add(List.of(doc1, doc2, doc3));
Awaitility.await()
.until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("Content").topK(5).build()),
hasSize(3));
// Complex filter expression: (type == 'A' AND priority > 1)
Filter.Expression priorityFilter = new Filter.Expression(Filter.ExpressionType.GT,
new Filter.Key("priority"), new Filter.Value(1));
Filter.Expression typeFilter = new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("type"),
new Filter.Value("A"));
Filter.Expression complexFilter = new Filter.Expression(Filter.ExpressionType.AND, typeFilter,
priorityFilter);
vectorStore.delete(complexFilter);
Awaitility.await()
.until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("Content").topK(5).build()),
hasSize(2));
var results = vectorStore
.similaritySearch(SearchRequest.builder().query("Content").topK(5).similarityThresholdAll().build());
assertThat(results).hasSize(2);
assertThat(results.stream().map(doc -> doc.getMetadata().get("type")).collect(Collectors.toList()))
.containsExactlyInAnyOrder("A", "B");
assertThat(results.stream().map(doc -> doc.getMetadata().get("priority")).collect(Collectors.toList()))
.containsExactlyInAnyOrder(1, 1);
});
}
@SpringBootConfiguration
@EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class })
public static class TestApplication {