Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add filter-based deletion to Redis vector store

Browse Source 
Add string-based filter deletion alongside the Filter.Expression-based deletion
for Redis vector store, providing consistent deletion capabilities with
other vector store implementations.

Key changes:
- Add delete(Filter.Expression) implementation using Redis FT.SEARCH and JSON.DEL
- Configure metadata fields properly to support numeric and tag operations
- Support both simple and complex filter expressions
- Handle Redis-specific JSON string responses in tests
- Add comprehensive integration tests for filter deletion cases

This maintains consistency with other vector store implementations while
utilizing Redis Search capabilities for efficient metadata-based deletion.

Signed-off-by: Soby Chacko <soby.chacko@broadcom.com>
This commit is contained in:
Soby Chacko
2025-01-28 12:02:30 -05:00
committed by Mark Pollack
parent 81d5618b3a
commit bca65de6ae
2 changed files with 134 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
vector-stores/spring-ai-redis-store/src/main/java/org/springframework/ai/vectorstore/redis/RedisVectorStore.java
View File

@@ -53,6 +53,7 @@ import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetri
import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.filter.Filter;
import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
@@ -296,6 +297,45 @@ public class RedisVectorStore extends AbstractObservationVectorStore implements
}
}
@Override
protected void doDelete(Filter.Expression filterExpression) {
Assert.notNull(filterExpression, "Filter expression must not be null");
try {
String filterStr = this.filterExpressionConverter.convertExpression(filterExpression);
List<String> matchingIds = new ArrayList<>();
SearchResult searchResult = this.jedis.ftSearch(this.indexName, filterStr);
for (redis.clients.jedis.search.Document doc : searchResult.getDocuments()) {
String docId = doc.getId();
matchingIds.add(docId.replace(key(""), "")); // Remove the key prefix to
// get original ID
}
if (!matchingIds.isEmpty()) {
try (Pipeline pipeline = this.jedis.pipelined()) {
for (String id : matchingIds) {
pipeline.jsonDel(key(id));
}
List<Object> responses = pipeline.syncAndReturnAll();
Optional<Object> errResponse = responses.stream().filter(Predicate.not(RESPONSE_DEL_OK)).findAny();
if (errResponse.isPresent()) {
logger.error(() -> "Could not delete document: " + errResponse.get());
throw new IllegalStateException("Failed to delete some documents");
}
}
logger.debug(() -> "Deleted " + matchingIds.size() + " documents matching filter expression");
}
}
catch (Exception e) {
logger.error(e, () -> "Failed to delete documents by filter");
throw new IllegalStateException("Failed to delete documents by filter", e);
}
}
@Override
public List<Document> doSimilaritySearch(SearchRequest request) {

96
vector-stores/spring-ai-redis-store/src/test/java/org/springframework/ai/vectorstore/redis/RedisVectorStoreIT.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2024 the original author or authors.
* Copyright 2023-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import com.redis.testcontainers.RedisStackContainer;
import org.junit.jupiter.api.BeforeEach;
@@ -36,6 +37,7 @@ import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.transformers.TransformersEmbeddingModel;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.filter.Filter;
import org.springframework.ai.vectorstore.redis.RedisVectorStore.MetadataField;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -53,6 +55,7 @@ import static org.assertj.core.api.Assertions.assertThat;
* @author Julien Ruaux
* @author Eddú Meléndez
* @author Thomas Vitale
* @author Soby Chacko
*/
@Testcontainers
class RedisVectorStoreIT {
@@ -260,6 +263,90 @@ class RedisVectorStoreIT {
});
}
@Test
void deleteByFilter() {
this.contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2020));
var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "NL"));
var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2023));
vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
Filter.Expression filterExpression = new Filter.Expression(Filter.ExpressionType.EQ,
new Filter.Key("country"), new Filter.Value("BG"));
vectorStore.delete(filterExpression);
List<Document> results = vectorStore
.similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build());
assertThat(results).hasSize(1);
assertThat(results.get(0).getMetadata()).containsEntry("country", "NL");
});
}
@Test
void deleteWithStringFilterExpression() {
this.contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2020));
var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "NL"));
var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2023));
vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
vectorStore.delete("country == 'BG'");
List<Document> results = vectorStore
.similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build());
assertThat(results).hasSize(1);
assertThat(results.get(0).getMetadata()).containsEntry("country", "NL");
});
}
@Test
void deleteWithComplexFilterExpression() {
this.contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
var doc1 = new Document("Content 1", Map.of("type", "A", "priority", 1));
var doc2 = new Document("Content 2", Map.of("type", "A", "priority", 2));
var doc3 = new Document("Content 3", Map.of("type", "B", "priority", 1));
vectorStore.add(List.of(doc1, doc2, doc3));
// Complex filter expression: (type == 'A' AND priority > 1)
Filter.Expression priorityFilter = new Filter.Expression(Filter.ExpressionType.GT,
new Filter.Key("priority"), new Filter.Value(1));
Filter.Expression typeFilter = new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("type"),
new Filter.Value("A"));
Filter.Expression complexFilter = new Filter.Expression(Filter.ExpressionType.AND, typeFilter,
priorityFilter);
vectorStore.delete(complexFilter);
var results = vectorStore
.similaritySearch(SearchRequest.builder().query("Content").topK(5).similarityThresholdAll().build());
assertThat(results).hasSize(2);
assertThat(results.stream().map(doc -> doc.getMetadata().get("type")).collect(Collectors.toList()))
.containsExactlyInAnyOrder("A", "B");
assertThat(results.stream()
.map(doc -> Integer.parseInt(doc.getMetadata().get("priority").toString()))
.collect(Collectors.toList())).containsExactlyInAnyOrder(1, 1);
});
}
@SpringBootConfiguration
@EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class })
public static class TestApplication {
@@ -271,7 +358,12 @@ class RedisVectorStoreIT {
.builder(new JedisPooled(jedisConnectionFactory.getHostName(), jedisConnectionFactory.getPort()),
embeddingModel)
.metadataFields(MetadataField.tag("meta1"), MetadataField.tag("meta2"), MetadataField.tag("country"),
MetadataField.numeric("year"))
MetadataField.numeric("year"), MetadataField.numeric("priority"), // Add
// priority
// as
// numeric
MetadataField.tag("type") // Add type as tag
)
.initializeSchema(true)
.build();
}