diff --git a/pom.xml b/pom.xml
index 767db5655..2198a8da0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,6 +33,7 @@
spring-ai-docsspring-ai-bomspring-ai-core
+ spring-ai-vector-storespring-ai-testspring-ai-spring-boot-autoconfigurespring-ai-retry
diff --git a/spring-ai-vector-store/pom.xml b/spring-ai-vector-store/pom.xml
new file mode 100644
index 000000000..ddad91a63
--- /dev/null
+++ b/spring-ai-vector-store/pom.xml
@@ -0,0 +1,100 @@
+
+
+
+
+ 4.0.0
+
+ org.springframework.ai
+ spring-ai
+ 1.0.0-SNAPSHOT
+
+ spring-ai-vector-store
+ jar
+ Spring AI Vector Store
+ Spring AI Vector Store APIs
+ https://github.com/spring-projects/spring-ai
+
+
+ https://github.com/spring-projects/spring-ai
+ git://github.com/spring-projects/spring-ai.git
+ git@github.com:spring-projects/spring-ai.git
+
+
+
+
+
+
+
+
+
+ io.micrometer
+ micrometer-core
+
+
+
+ io.micrometer
+ context-propagation
+
+
+
+ io.micrometer
+ micrometer-tracing-bridge-otel
+ true
+
+
+
+
+ org.springframework.boot
+ spring-boot-starter-test
+ test
+
+
+
+
+
+
+ antlr4
+
+ false
+
+
+
+
+ org.antlr
+ antlr4-maven-plugin
+ ${antlr.version}
+
+ ${basedir}/src/main/resources/antlr4
+ ${basedir}/src/main/java
+
+ true
+
+
+
+
+ antlr4
+
+
+
+
+
+
+
+
+
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreSimilarityTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreSimilarityTests.java
new file mode 100644
index 000000000..499d0269a
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreSimilarityTests.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.springframework.ai.document.Document;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Ilayaperumal Gopinathan
+ * @author Thomas Vitale
+ */
+public class SimpleVectorStoreSimilarityTests {
+
+ @Test
+ public void testSimilarity() {
+ Map metadata = new HashMap<>();
+ metadata.put("foo", "bar");
+ float[] testEmbedding = new float[] { 1.0f, 2.0f, 3.0f };
+
+ SimpleVectorStoreContent storeContent = new SimpleVectorStoreContent("1", "hello, how are you?", metadata,
+ testEmbedding);
+ Document document = storeContent.toDocument(0.6);
+ assertThat(document).isNotNull();
+ assertThat(document.getId()).isEqualTo("1");
+ assertThat(document.getContent()).isEqualTo("hello, how are you?");
+ assertThat(document.getMetadata().get("foo")).isEqualTo("bar");
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreTests.java
new file mode 100644
index 000000000..4e81eb5d3
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreTests.java
@@ -0,0 +1,260 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.CleanupMode;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.core.io.Resource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class SimpleVectorStoreTests {
+
+ @TempDir(cleanup = CleanupMode.ON_SUCCESS)
+ Path tempDir;
+
+ private SimpleVectorStore vectorStore;
+
+ private EmbeddingModel mockEmbeddingModel;
+
+ @BeforeEach
+ void setUp() {
+ this.mockEmbeddingModel = mock(EmbeddingModel.class);
+ when(this.mockEmbeddingModel.dimensions()).thenReturn(3);
+ when(this.mockEmbeddingModel.embed(any(String.class))).thenReturn(new float[] { 0.1f, 0.2f, 0.3f });
+ when(this.mockEmbeddingModel.embed(any(Document.class))).thenReturn(new float[] { 0.1f, 0.2f, 0.3f });
+ this.vectorStore = new SimpleVectorStore(this.mockEmbeddingModel);
+ }
+
+ @Test
+ void shouldAddAndRetrieveDocument() {
+ Document doc = Document.builder().id("1").text("test content").metadata(Map.of("key", "value")).build();
+
+ this.vectorStore.add(List.of(doc));
+
+ List results = this.vectorStore.similaritySearch("test content");
+ assertThat(results).hasSize(1).first().satisfies(result -> {
+ assertThat(result.getId()).isEqualTo("1");
+ assertThat(result.getContent()).isEqualTo("test content");
+ assertThat(result.getMetadata()).containsEntry("key", "value");
+ });
+ }
+
+ @Test
+ void shouldAddMultipleDocuments() {
+ List docs = Arrays.asList(Document.builder().id("1").text("first").build(),
+ Document.builder().id("2").text("second").build());
+
+ this.vectorStore.add(docs);
+
+ List results = this.vectorStore.similaritySearch("first");
+ assertThat(results).hasSize(2).extracting(Document::getId).containsExactlyInAnyOrder("1", "2");
+ }
+
+ @Test
+ void shouldHandleEmptyDocumentList() {
+ assertThatThrownBy(() -> this.vectorStore.add(Collections.emptyList()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Documents list cannot be empty");
+ }
+
+ @Test
+ void shouldHandleNullDocumentList() {
+ assertThatThrownBy(() -> this.vectorStore.add(null)).isInstanceOf(NullPointerException.class)
+ .hasMessage("Documents list cannot be null");
+ }
+
+ @Test
+ void shouldDeleteDocuments() {
+ Document doc = Document.builder().id("1").text("test content").build();
+
+ this.vectorStore.add(List.of(doc));
+ assertThat(this.vectorStore.similaritySearch("test")).hasSize(1);
+
+ this.vectorStore.delete(List.of("1"));
+ assertThat(this.vectorStore.similaritySearch("test")).isEmpty();
+ }
+
+ @Test
+ void shouldHandleDeleteOfNonexistentDocument() {
+ this.vectorStore.delete(List.of("nonexistent-id"));
+ // Should not throw exception and return true
+ assertThat(this.vectorStore.delete(List.of("nonexistent-id")).get()).isTrue();
+ }
+
+ @Test
+ void shouldPerformSimilaritySearchWithThreshold() {
+ // Configure mock to return different embeddings for different queries
+ when(this.mockEmbeddingModel.embed("query")).thenReturn(new float[] { 0.9f, 0.9f, 0.9f });
+
+ Document doc = Document.builder().id("1").text("test content").build();
+
+ this.vectorStore.add(List.of(doc));
+
+ SearchRequest request = SearchRequest.query("query").withSimilarityThreshold(0.99f).withTopK(5);
+
+ List results = this.vectorStore.similaritySearch(request);
+ assertThat(results).isEmpty();
+ }
+
+ @Test
+ void shouldSaveAndLoadVectorStore() throws IOException {
+ Document doc = Document.builder()
+ .id("1")
+ .text("test content")
+ .metadata(new HashMap<>(Map.of("key", "value")))
+ .build();
+
+ this.vectorStore.add(List.of(doc));
+
+ File saveFile = this.tempDir.resolve("vector-store.json").toFile();
+ this.vectorStore.save(saveFile);
+
+ SimpleVectorStore loadedStore = new SimpleVectorStore(this.mockEmbeddingModel);
+ loadedStore.load(saveFile);
+
+ List results = loadedStore.similaritySearch("test content");
+ assertThat(results).hasSize(1).first().satisfies(result -> {
+ assertThat(result.getId()).isEqualTo("1");
+ assertThat(result.getContent()).isEqualTo("test content");
+ assertThat(result.getMetadata()).containsEntry("key", "value");
+ });
+ }
+
+ @Test
+ void shouldHandleLoadFromInvalidResource() throws IOException {
+ Resource mockResource = mock(Resource.class);
+ when(mockResource.getInputStream()).thenThrow(new IOException("Resource not found"));
+
+ assertThatThrownBy(() -> this.vectorStore.load(mockResource)).isInstanceOf(RuntimeException.class)
+ .hasCauseInstanceOf(IOException.class)
+ .hasMessageContaining("Resource not found");
+ }
+
+ @Test
+ void shouldHandleSaveToInvalidLocation() {
+ File invalidFile = new File("/invalid/path/file.json");
+
+ assertThatThrownBy(() -> this.vectorStore.save(invalidFile)).isInstanceOf(RuntimeException.class)
+ .hasCauseInstanceOf(IOException.class);
+ }
+
+ @Test
+ void shouldHandleConcurrentOperations() throws InterruptedException {
+ int numThreads = 10;
+ Thread[] threads = new Thread[numThreads];
+
+ for (int i = 0; i < numThreads; i++) {
+ final String id = String.valueOf(i);
+ threads[i] = new Thread(() -> {
+ Document doc = Document.builder().id(id).text("content " + id).build();
+ this.vectorStore.add(List.of(doc));
+ });
+ threads[i].start();
+ }
+
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ SearchRequest request = SearchRequest.query("test").withTopK(numThreads);
+
+ List results = this.vectorStore.similaritySearch(request);
+
+ assertThat(results).hasSize(numThreads);
+
+ // Verify all documents were properly added
+ Set resultIds = results.stream().map(Document::getId).collect(Collectors.toSet());
+
+ Set expectedIds = new java.util.HashSet<>();
+ for (int i = 0; i < numThreads; i++) {
+ expectedIds.add(String.valueOf(i));
+ }
+
+ assertThat(resultIds).containsExactlyInAnyOrderElementsOf(expectedIds);
+
+ // Verify content integrity
+ results.forEach(doc -> assertThat(doc.getContent()).isEqualTo("content " + doc.getId()));
+ }
+
+ @Test
+ void shouldRejectInvalidSimilarityThreshold() {
+ assertThatThrownBy(() -> SearchRequest.query("test").withSimilarityThreshold(2.0f))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Similarity threshold must be in [0,1] range.");
+ }
+
+ @Test
+ void shouldRejectNegativeTopK() {
+ assertThatThrownBy(() -> SearchRequest.query("test").withTopK(-1)).isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("TopK should be positive.");
+ }
+
+ @Test
+ void shouldHandleCosineSimilarityEdgeCases() {
+ float[] zeroVector = new float[] { 0f, 0f, 0f };
+ float[] normalVector = new float[] { 1f, 1f, 1f };
+
+ assertThatThrownBy(() -> SimpleVectorStore.EmbeddingMath.cosineSimilarity(zeroVector, normalVector))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Vectors cannot have zero norm");
+ }
+
+ @Test
+ void shouldHandleVectorLengthMismatch() {
+ float[] vector1 = new float[] { 1f, 2f };
+ float[] vector2 = new float[] { 1f, 2f, 3f };
+
+ assertThatThrownBy(() -> SimpleVectorStore.EmbeddingMath.cosineSimilarity(vector1, vector2))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Vectors lengths must be equal");
+ }
+
+ @Test
+ void shouldHandleNullVectors() {
+ float[] vector = new float[] { 1f, 2f, 3f };
+
+ assertThatThrownBy(() -> SimpleVectorStore.EmbeddingMath.cosineSimilarity(null, vector))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("Vectors must not be null");
+
+ assertThatThrownBy(() -> SimpleVectorStore.EmbeddingMath.cosineSimilarity(vector, null))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("Vectors must not be null");
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilderTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilderTests.java
new file mode 100644
index 000000000..12084d007
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilderTests.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.filter;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.vectorstore.filter.Filter.Expression;
+import org.springframework.ai.vectorstore.filter.Filter.Group;
+import org.springframework.ai.vectorstore.filter.Filter.Key;
+import org.springframework.ai.vectorstore.filter.Filter.Value;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.AND;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.EQ;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.GTE;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.IN;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NE;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NIN;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NOT;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.OR;
+
+/**
+ * @author Christian Tzolov
+ */
+public class FilterExpressionBuilderTests {
+
+ FilterExpressionBuilder b = new FilterExpressionBuilder();
+
+ @Test
+ public void testEQ() {
+ // country == "BG"
+ assertThat(this.b.eq("country", "BG").build())
+ .isEqualTo(new Expression(EQ, new Key("country"), new Value("BG")));
+ }
+
+ @Test
+ public void tesEqAndGte() {
+ // genre == "drama" AND year >= 2020
+ Expression exp = this.b.and(this.b.eq("genre", "drama"), this.b.gte("year", 2020)).build();
+ assertThat(exp).isEqualTo(new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")),
+ new Expression(GTE, new Key("year"), new Value(2020))));
+ }
+
+ @Test
+ public void testIn() {
+ // genre in ["comedy", "documentary", "drama"]
+ var exp = this.b.in("genre", "comedy", "documentary", "drama").build();
+ assertThat(exp)
+ .isEqualTo(new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama"))));
+ }
+
+ @Test
+ public void testNe() {
+ // year >= 2020 OR country == "BG" AND city != "Sofia"
+ var exp = this.b
+ .and(this.b.or(this.b.gte("year", 2020), this.b.eq("country", "BG")), this.b.ne("city", "Sofia"))
+ .build();
+
+ assertThat(exp).isEqualTo(new Expression(AND,
+ new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)),
+ new Expression(EQ, new Key("country"), new Value("BG"))),
+ new Expression(NE, new Key("city"), new Value("Sofia"))));
+ }
+
+ @Test
+ public void testGroup() {
+ // (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"]
+ var exp = this.b
+ .and(this.b.group(this.b.or(this.b.gte("year", 2020), this.b.eq("country", "BG"))),
+ this.b.nin("city", "Sofia", "Plovdiv"))
+ .build();
+
+ assertThat(exp).isEqualTo(new Expression(AND,
+ new Group(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)),
+ new Expression(EQ, new Key("country"), new Value("BG")))),
+ new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Plovdiv")))));
+ }
+
+ @Test
+ public void tesIn2() {
+ // isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]
+ var exp = this.b
+ .and(this.b.and(this.b.eq("isOpen", true), this.b.gte("year", 2020)),
+ this.b.in("country", "BG", "NL", "US"))
+ .build();
+
+ assertThat(exp).isEqualTo(new Expression(AND,
+ new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)),
+ new Expression(GTE, new Key("year"), new Value(2020))),
+ new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))));
+ }
+
+ @Test
+ public void tesNot() {
+ // isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]
+ var exp = this.b.not(this.b.and(this.b.and(this.b.eq("isOpen", true), this.b.gte("year", 2020)),
+ this.b.in("country", "BG", "NL", "US")))
+ .build();
+
+ assertThat(exp).isEqualTo(new Expression(NOT,
+ new Expression(AND,
+ new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)),
+ new Expression(GTE, new Key("year"), new Value(2020))),
+ new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))),
+ null));
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java
new file mode 100644
index 000000000..218f29c97
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java
@@ -0,0 +1,201 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.filter;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.vectorstore.filter.Filter.Expression;
+import org.springframework.ai.vectorstore.filter.Filter.Group;
+import org.springframework.ai.vectorstore.filter.Filter.Key;
+import org.springframework.ai.vectorstore.filter.Filter.Value;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.AND;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.EQ;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.GTE;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.IN;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.LTE;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NE;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NIN;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NOT;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.OR;
+
+/**
+ * @author Christian Tzolov
+ */
+public class FilterExpressionTextParserTests {
+
+ FilterExpressionTextParser parser = new FilterExpressionTextParser();
+
+ @Test
+ public void testEQ() {
+ // country == "BG"
+ Expression exp = this.parser.parse("country == 'BG'");
+ assertThat(exp).isEqualTo(new Expression(EQ, new Key("country"), new Value("BG")));
+
+ assertThat(this.parser.getCache().get("WHERE " + "country == 'BG'")).isEqualTo(exp);
+ }
+
+ @Test
+ public void tesEqAndGte() {
+ // genre == "drama" AND year >= 2020
+ Expression exp = this.parser.parse("genre == 'drama' && year >= 2020");
+ assertThat(exp).isEqualTo(new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")),
+ new Expression(GTE, new Key("year"), new Value(2020))));
+
+ assertThat(this.parser.getCache().get("WHERE " + "genre == 'drama' && year >= 2020")).isEqualTo(exp);
+ }
+
+ @Test
+ public void tesIn() {
+ // genre in ["comedy", "documentary", "drama"]
+ Expression exp = this.parser.parse("genre in ['comedy', 'documentary', 'drama']");
+ assertThat(exp)
+ .isEqualTo(new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama"))));
+
+ assertThat(this.parser.getCache().get("WHERE " + "genre in ['comedy', 'documentary', 'drama']")).isEqualTo(exp);
+ }
+
+ @Test
+ public void testNe() {
+ // year >= 2020 OR country == "BG" AND city != "Sofia"
+ Expression exp = this.parser.parse("year >= 2020 OR country == \"BG\" AND city != \"Sofia\"");
+ assertThat(exp).isEqualTo(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)),
+ new Expression(AND, new Expression(EQ, new Key("country"), new Value("BG")),
+ new Expression(NE, new Key("city"), new Value("Sofia")))));
+
+ assertThat(this.parser.getCache().get("WHERE " + "year >= 2020 OR country == \"BG\" AND city != \"Sofia\""))
+ .isEqualTo(exp);
+ }
+
+ @Test
+ public void testGroup() {
+ // (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"]
+ Expression exp = this.parser.parse("(year >= 2020 OR country == \"BG\") AND city NIN [\"Sofia\", \"Plovdiv\"]");
+
+ assertThat(exp).isEqualTo(new Expression(AND,
+ new Group(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)),
+ new Expression(EQ, new Key("country"), new Value("BG")))),
+ new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Plovdiv")))));
+
+ assertThat(this.parser.getCache()
+ .get("WHERE " + "(year >= 2020 OR country == \"BG\") AND city NIN [\"Sofia\", \"Plovdiv\"]"))
+ .isEqualTo(exp);
+ }
+
+ @Test
+ public void tesBoolean() {
+ // isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]
+ Expression exp = this.parser.parse("isOpen == true AND year >= 2020 AND country IN [\"BG\", \"NL\", \"US\"]");
+
+ assertThat(exp).isEqualTo(new Expression(AND,
+ new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)),
+ new Expression(GTE, new Key("year"), new Value(2020))),
+ new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))));
+ assertThat(this.parser.getCache()
+ .get("WHERE " + "isOpen == true AND year >= 2020 AND country IN [\"BG\", \"NL\", \"US\"]")).isEqualTo(exp);
+ }
+
+ @Test
+ public void tesNot() {
+ // NOT(isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"])
+ Expression exp = this.parser
+ .parse("not(isOpen == true AND year >= 2020 AND country IN [\"BG\", \"NL\", \"US\"])");
+
+ assertThat(exp).isEqualTo(new Expression(NOT,
+ new Group(new Expression(AND,
+ new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)),
+ new Expression(GTE, new Key("year"), new Value(2020))),
+ new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US"))))),
+ null));
+
+ assertThat(this.parser.getCache()
+ .get("WHERE " + "not(isOpen == true AND year >= 2020 AND country IN [\"BG\", \"NL\", \"US\"])"))
+ .isEqualTo(exp);
+ }
+
+ @Test
+ public void tesNotNin() {
+ // NOT(country NOT IN ["BG", "NL", "US"])
+ Expression exp = this.parser.parse("not(country NOT IN [\"BG\", \"NL\", \"US\"])");
+
+ assertThat(exp).isEqualTo(new Expression(NOT,
+ new Group(new Expression(NIN, new Key("country"), new Value(List.of("BG", "NL", "US")))), null));
+ }
+
+ @Test
+ public void tesNotNin2() {
+ // NOT country NOT IN ["BG", "NL", "US"]
+ Expression exp = this.parser.parse("NOT country NOT IN [\"BG\", \"NL\", \"US\"]");
+
+ assertThat(exp).isEqualTo(new Expression(NOT,
+ new Expression(NIN, new Key("country"), new Value(List.of("BG", "NL", "US"))), null));
+ }
+
+ @Test
+ public void tesNestedNot() {
+ // NOT(isOpen == true AND year >= 2020 AND NOT(country IN ["BG", "NL", "US"]))
+ Expression exp = this.parser
+ .parse("not(isOpen == true AND year >= 2020 AND NOT(country IN [\"BG\", \"NL\", \"US\"]))");
+
+ assertThat(exp).isEqualTo(new Expression(NOT,
+ new Group(new Expression(AND,
+ new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)),
+ new Expression(GTE, new Key("year"), new Value(2020))),
+ new Expression(NOT,
+ new Group(new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))),
+ null))),
+ null));
+
+ assertThat(this.parser.getCache()
+ .get("WHERE " + "not(isOpen == true AND year >= 2020 AND NOT(country IN [\"BG\", \"NL\", \"US\"]))"))
+ .isEqualTo(exp);
+ }
+
+ @Test
+ public void testDecimal() {
+ // temperature >= -15.6 && temperature <= +20.13
+ String expText = "temperature >= -15.6 && temperature <= +20.13";
+ Expression exp = this.parser.parse(expText);
+
+ assertThat(exp).isEqualTo(new Expression(AND, new Expression(GTE, new Key("temperature"), new Value(-15.6)),
+ new Expression(LTE, new Key("temperature"), new Value(20.13))));
+
+ assertThat(this.parser.getCache().get("WHERE " + expText)).isEqualTo(exp);
+ }
+
+ @Test
+ public void testIdentifiers() {
+ Expression exp = this.parser.parse("'country.1' == 'BG'");
+ assertThat(exp).isEqualTo(new Expression(EQ, new Key("'country.1'"), new Value("BG")));
+
+ exp = this.parser.parse("'country_1_2_3' == 'BG'");
+ assertThat(exp).isEqualTo(new Expression(EQ, new Key("'country_1_2_3'"), new Value("BG")));
+
+ exp = this.parser.parse("\"country 1 2 3\" == 'BG'");
+ assertThat(exp).isEqualTo(new Expression(EQ, new Key("\"country 1 2 3\""), new Value("BG")));
+ }
+
+ @Test
+ public void testUnescapedIdentifierWithUnderscores() {
+ Expression exp = this.parser.parse("file_name == 'medicaid-wa-faqs.pdf'");
+ assertThat(exp).isEqualTo(new Expression(EQ, new Key("file_name"), new Value("medicaid-wa-faqs.pdf")));
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterHelperTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterHelperTests.java
new file mode 100644
index 000000000..df793ecf0
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterHelperTests.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.filter;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.vectorstore.filter.Filter.Expression;
+import org.springframework.ai.vectorstore.filter.Filter.ExpressionType;
+import org.springframework.ai.vectorstore.filter.Filter.Key;
+import org.springframework.ai.vectorstore.filter.Filter.Value;
+import org.springframework.ai.vectorstore.filter.converter.PrintFilterExpressionConverter;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Christian Tzolov
+ */
+public class FilterHelperTests {
+
+ @Test
+ public void negateEQ() {
+ assertThat(new FilterExpressionTextParser().parse("NOT key == 'UK' ")).isEqualTo(new Filter.Expression(
+ ExpressionType.NOT, new Filter.Expression(ExpressionType.EQ, new Key("key"), new Value("UK")), null));
+
+ assertThat(FilterHelper.negate(new FilterExpressionTextParser().parse("NOT key == 'UK' ")))
+ .isEqualTo(new Filter.Expression(ExpressionType.NE, new Key("key"), new Value("UK")));
+
+ assertThat(FilterHelper.negate(new FilterExpressionTextParser().parse("NOT (key == 'UK') ")))
+ .isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.NE, new Key("key"), new Value("UK"))));
+ }
+
+ @Test
+ public void negateNE() {
+ var exp = new FilterExpressionTextParser().parse("NOT key != 'UK' ");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Expression(ExpressionType.EQ, new Key("key"), new Value("UK")));
+
+ }
+
+ @Test
+ public void negateGT() {
+ var exp = new FilterExpressionTextParser().parse("NOT key > 13 ");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Expression(ExpressionType.LTE, new Key("key"), new Value(13)));
+
+ }
+
+ @Test
+ public void negateGTE() {
+ var exp = new FilterExpressionTextParser().parse("NOT key >= 13 ");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(13)));
+ }
+
+ @Test
+ public void negateLT() {
+ var exp = new FilterExpressionTextParser().parse("NOT key < 13 ");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Expression(ExpressionType.GTE, new Key("key"), new Value(13)));
+ }
+
+ @Test
+ public void negateLTE() {
+ var exp = new FilterExpressionTextParser().parse("NOT key <= 13 ");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Expression(ExpressionType.GT, new Key("key"), new Value(13)));
+ }
+
+ @Test
+ public void negateIN() {
+ var exp = new FilterExpressionTextParser().parse("NOT key IN [11, 12, 13] ");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Expression(ExpressionType.NIN, new Key("key"), new Value(List.of(11, 12, 13))));
+ }
+
+ @Test
+ public void negateNIN() {
+ var exp = new FilterExpressionTextParser().parse("NOT key NIN [11, 12, 13] ");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Expression(ExpressionType.IN, new Key("key"), new Value(List.of(11, 12, 13))));
+ }
+
+ @Test
+ public void negateNIN2() {
+ var exp = new FilterExpressionTextParser().parse("NOT key NOT IN [11, 12, 13] ");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Expression(ExpressionType.IN, new Key("key"), new Value(List.of(11, 12, 13))));
+ }
+
+ @Test
+ public void negateAND() {
+ var exp = new FilterExpressionTextParser().parse("NOT(key >= 11 AND key < 13)");
+ assertThat(FilterHelper.negate(exp)).isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.OR,
+ new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(11)),
+ new Filter.Expression(ExpressionType.GTE, new Key("key"), new Value(13)))));
+ }
+
+ @Test
+ public void negateOR() {
+ var exp = new FilterExpressionTextParser().parse("NOT(key >= 11 OR key < 13)");
+ assertThat(FilterHelper.negate(exp)).isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.AND,
+ new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(11)),
+ new Filter.Expression(ExpressionType.GTE, new Key("key"), new Value(13)))));
+ }
+
+ @Test
+ public void negateNot() {
+ var exp = new FilterExpressionTextParser().parse("NOT NOT(key >= 11)");
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(11))));
+ }
+
+ @Test
+ public void negateNestedNot() {
+ var exp = new FilterExpressionTextParser().parse("NOT(NOT(key >= 11))");
+ assertThat(exp).isEqualTo(
+ new Filter.Expression(ExpressionType.NOT, new Filter.Group(new Filter.Expression(ExpressionType.NOT,
+ new Filter.Group(new Filter.Expression(ExpressionType.GTE, new Key("key"), new Value(11)))))));
+
+ assertThat(FilterHelper.negate(exp))
+ .isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(11))));
+ }
+
+ @Test
+ public void expandIN() {
+ var exp = new FilterExpressionTextParser().parse("key IN [11, 12, 13] ");
+ assertThat(new InNinTestConverter().convertExpression(exp)).isEqualTo("key EQ 11 OR key EQ 12 OR key EQ 13");
+ }
+
+ @Test
+ public void expandNIN() {
+ var exp1 = new FilterExpressionTextParser().parse("key NIN [11, 12, 13] ");
+ var exp2 = new FilterExpressionTextParser().parse("key NOT IN [11, 12, 13] ");
+ assertThat(exp1).isEqualTo(exp2);
+ assertThat(new InNinTestConverter().convertExpression(exp1)).isEqualTo("key NE 11 AND key NE 12 AND key NE 13");
+ }
+
+ private static class InNinTestConverter extends PrintFilterExpressionConverter {
+
+ @Override
+ public void doExpression(Expression expression, StringBuilder context) {
+ if (expression.type() == ExpressionType.IN) {
+ FilterHelper.expandIn(expression, context, this);
+ }
+ else if (expression.type() == ExpressionType.NIN) {
+ FilterHelper.expandNin(expression, context, this);
+ }
+ else {
+ super.doExpression(expression, context);
+ }
+ }
+
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/SearchRequestTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/SearchRequestTests.java
new file mode 100644
index 000000000..acdd0b3ed
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/SearchRequestTests.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.filter;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.vectorstore.SearchRequest;
+import org.springframework.ai.vectorstore.filter.FilterExpressionTextParser.FilterExpressionParseException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * @author Christian Tzolov
+ */
+public class SearchRequestTests {
+
+ @Test
+ public void createDefaults() {
+ var emptyRequest = SearchRequest.defaults();
+ assertThat(emptyRequest.getQuery()).isEqualTo("");
+ checkDefaults(emptyRequest);
+ }
+
+ @Test
+ public void createQuery() {
+ var emptyRequest = SearchRequest.query("New Query");
+ assertThat(emptyRequest.getQuery()).isEqualTo("New Query");
+ checkDefaults(emptyRequest);
+ }
+
+ @Test
+ public void createFrom() {
+ var originalRequest = SearchRequest.query("New Query")
+ .withTopK(696)
+ .withSimilarityThreshold(0.678)
+ .withFilterExpression("country == 'NL'");
+
+ var newRequest = SearchRequest.from(originalRequest);
+
+ assertThat(newRequest).isNotSameAs(originalRequest);
+ assertThat(newRequest.getQuery()).isEqualTo(originalRequest.getQuery());
+ assertThat(newRequest.getTopK()).isEqualTo(originalRequest.getTopK());
+ assertThat(newRequest.getFilterExpression()).isEqualTo(originalRequest.getFilterExpression());
+ assertThat(newRequest.getSimilarityThreshold()).isEqualTo(originalRequest.getSimilarityThreshold());
+ }
+
+ @Test
+ public void withQuery() {
+ var emptyRequest = SearchRequest.defaults();
+ assertThat(emptyRequest.getQuery()).isEqualTo("");
+
+ emptyRequest.withQuery("New Query");
+ assertThat(emptyRequest.getQuery()).isEqualTo("New Query");
+ }
+
+ @Test
+ public void withSimilarityThreshold() {
+ var request = SearchRequest.query("Test").withSimilarityThreshold(0.678);
+ assertThat(request.getSimilarityThreshold()).isEqualTo(0.678);
+
+ request.withSimilarityThreshold(0.9);
+ assertThat(request.getSimilarityThreshold()).isEqualTo(0.9);
+
+ assertThatThrownBy(() -> request.withSimilarityThreshold(-1)).isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Similarity threshold must be in [0,1] range.");
+
+ assertThatThrownBy(() -> request.withSimilarityThreshold(1.1)).isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Similarity threshold must be in [0,1] range.");
+
+ }
+
+ @Test
+ public void withTopK() {
+ var request = SearchRequest.query("Test").withTopK(66);
+ assertThat(request.getTopK()).isEqualTo(66);
+
+ request.withTopK(89);
+ assertThat(request.getTopK()).isEqualTo(89);
+
+ assertThatThrownBy(() -> request.withTopK(-1)).isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("TopK should be positive.");
+
+ }
+
+ @Test
+ public void withFilterExpression() {
+
+ var request = SearchRequest.query("Test").withFilterExpression("country == 'BG' && year >= 2022");
+ assertThat(request.getFilterExpression()).isEqualTo(new Filter.Expression(Filter.ExpressionType.AND,
+ new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("country"), new Filter.Value("BG")),
+ new Filter.Expression(Filter.ExpressionType.GTE, new Filter.Key("year"), new Filter.Value(2022))));
+ assertThat(request.hasFilterExpression()).isTrue();
+
+ request.withFilterExpression("active == true");
+ assertThat(request.getFilterExpression()).isEqualTo(
+ new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("active"), new Filter.Value(true)));
+ assertThat(request.hasFilterExpression()).isTrue();
+
+ request.withFilterExpression(new FilterExpressionBuilder().eq("country", "NL").build());
+ assertThat(request.getFilterExpression()).isEqualTo(
+ new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("country"), new Filter.Value("NL")));
+ assertThat(request.hasFilterExpression()).isTrue();
+
+ request.withFilterExpression((String) null);
+ assertThat(request.getFilterExpression()).isNull();
+ assertThat(request.hasFilterExpression()).isFalse();
+
+ request.withFilterExpression((Filter.Expression) null);
+ assertThat(request.getFilterExpression()).isNull();
+ assertThat(request.hasFilterExpression()).isFalse();
+
+ assertThatThrownBy(() -> request.withFilterExpression("FooBar"))
+ .isInstanceOf(FilterExpressionParseException.class)
+ .hasMessageContaining("Error: no viable alternative at input 'FooBar'");
+
+ }
+
+ private void checkDefaults(SearchRequest request) {
+ assertThat(request.getFilterExpression()).isNull();
+ assertThat(request.getSimilarityThreshold()).isEqualTo(SearchRequest.SIMILARITY_THRESHOLD_ACCEPT_ALL);
+ assertThat(request.getTopK()).isEqualTo(SearchRequest.DEFAULT_TOP_K);
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverterTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverterTests.java
new file mode 100644
index 000000000..9fc858aa1
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverterTests.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.filter.converter;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.vectorstore.filter.Filter.Expression;
+import org.springframework.ai.vectorstore.filter.Filter.Group;
+import org.springframework.ai.vectorstore.filter.Filter.Key;
+import org.springframework.ai.vectorstore.filter.Filter.Value;
+import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.AND;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.EQ;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.GTE;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.IN;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.LTE;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NE;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NIN;
+import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.OR;
+
+/**
+ * @author Christian Tzolov
+ */
+public class PineconeFilterExpressionConverterTests {
+
+ FilterExpressionConverter converter = new PineconeFilterExpressionConverter();
+
+ @Test
+ public void testEQ() {
+ // country == "BG"
+ String vectorExpr = this.converter.convertExpression(new Expression(EQ, new Key("country"), new Value("BG")));
+ assertThat(vectorExpr).isEqualTo("{\"country\": {\"$eq\": \"BG\"}}");
+ }
+
+ @Test
+ public void tesEqAndGte() {
+ // genre == "drama" AND year >= 2020
+ String vectorExpr = this.converter
+ .convertExpression(new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")),
+ new Expression(GTE, new Key("year"), new Value(2020))));
+ assertThat(vectorExpr)
+ .isEqualTo("{\"$and\": [{\"genre\": {\"$eq\": \"drama\"}},{\"year\": {\"$gte\": 2020}}]}");
+ }
+
+ @Test
+ public void tesIn() {
+ // genre in ["comedy", "documentary", "drama"]
+ String vectorExpr = this.converter.convertExpression(
+ new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama"))));
+ assertThat(vectorExpr).isEqualTo("{\"genre\": {\"$in\": [\"comedy\",\"documentary\",\"drama\"]}}");
+ }
+
+ @Test
+ public void testNe() {
+ // year >= 2020 OR country == "BG" AND city != "Sofia"
+ String vectorExpr = this.converter
+ .convertExpression(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)),
+ new Expression(AND, new Expression(EQ, new Key("country"), new Value("BG")),
+ new Expression(NE, new Key("city"), new Value("Sofia")))));
+ assertThat(vectorExpr).isEqualTo(
+ "{\"$or\": [{\"year\": {\"$gte\": 2020}},{\"$and\": [{\"country\": {\"$eq\": \"BG\"}},{\"city\": {\"$ne\": \"Sofia\"}}]}]}");
+ }
+
+ @Test
+ public void testGroup() {
+ // (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"]
+ String vectorExpr = this.converter.convertExpression(new Expression(AND,
+ new Group(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)),
+ new Expression(EQ, new Key("country"), new Value("BG")))),
+ new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Plovdiv")))));
+ assertThat(vectorExpr).isEqualTo(
+ "{\"$and\": [{\"$or\": [{\"year\": {\"$gte\": 2020}},{\"country\": {\"$eq\": \"BG\"}}]},{\"city\": {\"$nin\": [\"Sofia\",\"Plovdiv\"]}}]}");
+ }
+
+ @Test
+ public void tesBoolean() {
+ // isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]
+ String vectorExpr = this.converter.convertExpression(new Expression(AND,
+ new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)),
+ new Expression(GTE, new Key("year"), new Value(2020))),
+ new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))));
+
+ assertThat(vectorExpr).isEqualTo(
+ "{\"$and\": [{\"$and\": [{\"isOpen\": {\"$eq\": true}},{\"year\": {\"$gte\": 2020}}]},{\"country\": {\"$in\": [\"BG\",\"NL\",\"US\"]}}]}");
+ }
+
+ @Test
+ public void testDecimal() {
+ // temperature >= -15.6 && temperature <= +20.13
+ String vectorExpr = this.converter
+ .convertExpression(new Expression(AND, new Expression(GTE, new Key("temperature"), new Value(-15.6)),
+ new Expression(LTE, new Key("temperature"), new Value(20.13))));
+
+ assertThat(vectorExpr)
+ .isEqualTo("{\"$and\": [{\"temperature\": {\"$gte\": -15.6}},{\"temperature\": {\"$lte\": 20.13}}]}");
+ }
+
+ @Test
+ public void testComplexIdentifiers() {
+ String vectorExpr = this.converter
+ .convertExpression(new Expression(EQ, new Key("\"country 1 2 3\""), new Value("BG")));
+ assertThat(vectorExpr).isEqualTo("{\"country 1 2 3\": {\"$eq\": \"BG\"}}");
+
+ vectorExpr = this.converter.convertExpression(new Expression(EQ, new Key("'country 1 2 3'"), new Value("BG")));
+ assertThat(vectorExpr).isEqualTo("{\"country 1 2 3\": {\"$eq\": \"BG\"}}");
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConventionTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConventionTests.java
new file mode 100644
index 000000000..981ac04b1
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConventionTests.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.observation;
+
+import java.util.List;
+
+import io.micrometer.common.KeyValue;
+import io.micrometer.observation.Observation;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.observation.conventions.SpringAiKind;
+import org.springframework.ai.vectorstore.SearchRequest;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.HighCardinalityKeyNames;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.LowCardinalityKeyNames;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Unit tests for {@link DefaultVectorStoreObservationConvention}.
+ *
+ * @author Christian Tzolov
+ * @author Thomas Vitale
+ */
+class DefaultVectorStoreObservationConventionTests {
+
+ private final DefaultVectorStoreObservationConvention observationConvention = new DefaultVectorStoreObservationConvention();
+
+ @Test
+ void shouldHaveName() {
+ assertThat(this.observationConvention.getName())
+ .isEqualTo(DefaultVectorStoreObservationConvention.DEFAULT_NAME);
+ }
+
+ @Test
+ void shouldHaveContextualName() {
+ VectorStoreObservationContext observationContext = VectorStoreObservationContext
+ .builder("my-database", VectorStoreObservationContext.Operation.QUERY)
+ .build();
+ assertThat(this.observationConvention.getContextualName(observationContext)).isEqualTo("my-database query");
+ }
+
+ @Test
+ void supportsOnlyVectorStoreObservationContext() {
+ VectorStoreObservationContext observationContext = VectorStoreObservationContext
+ .builder("my-database", VectorStoreObservationContext.Operation.QUERY)
+ .build();
+ assertThat(this.observationConvention.supportsContext(observationContext)).isTrue();
+ assertThat(this.observationConvention.supportsContext(new Observation.Context())).isFalse();
+ }
+
+ @Test
+ void shouldHaveRequiredKeyValues() {
+ VectorStoreObservationContext observationContext = VectorStoreObservationContext
+ .builder("my_database", VectorStoreObservationContext.Operation.QUERY)
+ .build();
+ assertThat(this.observationConvention.getLowCardinalityKeyValues(observationContext)).contains(
+ KeyValue.of(LowCardinalityKeyNames.SPRING_AI_KIND.asString(), SpringAiKind.VECTOR_STORE.value()),
+ KeyValue.of(LowCardinalityKeyNames.DB_OPERATION_NAME.asString(), "query"),
+ KeyValue.of(LowCardinalityKeyNames.DB_SYSTEM.asString(), "my_database"));
+ }
+
+ @Test
+ void shouldHaveOptionalKeyValues() {
+ VectorStoreObservationContext observationContext = VectorStoreObservationContext
+ .builder("my-database", VectorStoreObservationContext.Operation.QUERY)
+ .withCollectionName("COLLECTION_NAME")
+ .withDimensions(696)
+ .withFieldName("FIELD_NAME")
+ .withNamespace("NAMESPACE")
+ .withSimilarityMetric("SIMILARITY_METRIC")
+ .withQueryRequest(SearchRequest.query("VDB QUERY").withFilterExpression("country == 'UK' && year >= 2020"))
+ .build();
+
+ List queryResponseDocs = List.of(new Document("doc1"), new Document("doc2"));
+
+ observationContext.setQueryResponse(queryResponseDocs);
+
+ assertThat(this.observationConvention.getLowCardinalityKeyValues(observationContext))
+ .contains(KeyValue.of(LowCardinalityKeyNames.DB_OPERATION_NAME.asString(),
+ VectorStoreObservationContext.Operation.QUERY.value));
+
+ // Optional, filter only added content
+ assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext))
+ .doesNotContain(KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_QUERY_RESPONSE_DOCUMENTS, "[doc1,doc2]"));
+
+ assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)).contains(
+ KeyValue.of(HighCardinalityKeyNames.DB_COLLECTION_NAME.asString(), "COLLECTION_NAME"),
+ KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_DIMENSION_COUNT.asString(), "696"),
+ KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_FIELD_NAME.asString(), "FIELD_NAME"),
+ KeyValue.of(HighCardinalityKeyNames.DB_NAMESPACE.asString(), "NAMESPACE"),
+ KeyValue.of(HighCardinalityKeyNames.DB_SEARCH_SIMILARITY_METRIC.asString(), "SIMILARITY_METRIC"),
+ KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_QUERY_CONTENT.asString(), "VDB QUERY"),
+ KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_QUERY_FILTER.asString(),
+ "Expression[type=AND, left=Expression[type=EQ, left=Key[key=country], right=Value[value=UK]], right=Expression[type=GTE, left=Key[key=year], right=Value[value=2020]]]"));
+ }
+
+ @Test
+ void shouldNotHaveKeyValuesWhenMissing() {
+ VectorStoreObservationContext observationContext = VectorStoreObservationContext
+ .builder("my-database", VectorStoreObservationContext.Operation.QUERY)
+ .build();
+
+ assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)
+ .stream()
+ .map(KeyValue::getKey)
+ .toList()).doesNotContain(HighCardinalityKeyNames.DB_COLLECTION_NAME.asString(),
+ HighCardinalityKeyNames.DB_VECTOR_DIMENSION_COUNT.asString(),
+ HighCardinalityKeyNames.DB_VECTOR_FIELD_NAME.asString(),
+ HighCardinalityKeyNames.DB_NAMESPACE.asString(),
+ HighCardinalityKeyNames.DB_SEARCH_SIMILARITY_METRIC.asString(),
+ HighCardinalityKeyNames.DB_VECTOR_QUERY_CONTENT.asString(),
+ HighCardinalityKeyNames.DB_VECTOR_QUERY_FILTER.asString());
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContextTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContextTests.java
new file mode 100644
index 000000000..6f6abd873
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContextTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.observation;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * Unit tests for {@link VectorStoreObservationContext}.
+ *
+ * @author Christian Tzolov
+ */
+class VectorStoreObservationContextTests {
+
+ @Test
+ void whenMandatoryFieldsThenReturn() {
+ var observationContext = VectorStoreObservationContext
+ .builder("db", VectorStoreObservationContext.Operation.ADD)
+ .build();
+ assertThat(observationContext).isNotNull();
+ }
+
+ @Test
+ void whenDbSystemIsNullThenThrow() {
+ assertThatThrownBy(() -> VectorStoreObservationContext.builder(null, "delete").build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("databaseSystem cannot be null or empty");
+ }
+
+ @Test
+ void whenOperationNameIsNullThenThrow() {
+ assertThatThrownBy(() -> VectorStoreObservationContext.builder("Db", "").build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("operationName cannot be null or empty");
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilterTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilterTests.java
new file mode 100644
index 000000000..ba7a37e05
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilterTests.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.observation;
+
+import java.util.List;
+
+import io.micrometer.common.KeyValue;
+import io.micrometer.observation.Observation;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.HighCardinalityKeyNames;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Unit tests for {@link VectorStoreQueryResponseObservationFilter}.
+ *
+ * @author Christian Tzolov
+ * @author Thomas Vitale
+ */
+class VectorStoreQueryResponseObservationFilterTests {
+
+ private final VectorStoreQueryResponseObservationFilter observationFilter = new VectorStoreQueryResponseObservationFilter();
+
+ @Test
+ void whenNotSupportedObservationContextThenReturnOriginalContext() {
+ var expectedContext = new Observation.Context();
+ var actualContext = this.observationFilter.map(expectedContext);
+
+ assertThat(actualContext).isEqualTo(expectedContext);
+ }
+
+ @Test
+ void whenEmptyQueryResponseThenReturnOriginalContext() {
+ var expectedContext = VectorStoreObservationContext.builder("db", VectorStoreObservationContext.Operation.ADD)
+ .build();
+
+ var actualContext = this.observationFilter.map(expectedContext);
+
+ assertThat(actualContext).isEqualTo(expectedContext);
+ }
+
+ @Test
+ void whenNonEmptyQueryResponseThenAugmentContext() {
+ var expectedContext = VectorStoreObservationContext.builder("db", VectorStoreObservationContext.Operation.ADD)
+ .build();
+
+ List queryResponseDocs = List.of(new Document("doc1"), new Document("doc2"));
+
+ expectedContext.setQueryResponse(queryResponseDocs);
+
+ var augmentedContext = this.observationFilter.map(expectedContext);
+
+ assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue
+ .of(HighCardinalityKeyNames.DB_VECTOR_QUERY_RESPONSE_DOCUMENTS.asString(), "[\"doc1\", \"doc2\"]"));
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandlerTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandlerTests.java
new file mode 100644
index 000000000..499c3cc02
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandlerTests.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.observation;
+
+import java.util.List;
+
+import io.micrometer.tracing.handler.TracingObservationHandler;
+import io.micrometer.tracing.otel.bridge.OtelCurrentTraceContext;
+import io.micrometer.tracing.otel.bridge.OtelTracer;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.observation.conventions.VectorStoreObservationAttributes;
+import org.springframework.ai.observation.conventions.VectorStoreObservationEventNames;
+import org.springframework.ai.observation.tracing.TracingHelper;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Unit tests for {@link VectorStoreQueryResponseObservationHandler}.
+ *
+ * @author Thomas Vitale
+ */
+class VectorStoreQueryResponseObservationHandlerTests {
+
+ @Test
+ void whenCompletionWithTextThenSpanEvent() {
+ var observationContext = VectorStoreObservationContext
+ .builder("db", VectorStoreObservationContext.Operation.ADD)
+ .withQueryResponse(List.of(new Document("hello"), new Document("other-side")))
+ .build();
+ var sdkTracer = SdkTracerProvider.builder().build().get("test");
+ var otelTracer = new OtelTracer(sdkTracer, new OtelCurrentTraceContext(), null);
+ var span = otelTracer.nextSpan();
+ var tracingContext = new TracingObservationHandler.TracingContext();
+ tracingContext.setSpan(span);
+ observationContext.put(TracingObservationHandler.TracingContext.class, tracingContext);
+
+ new VectorStoreQueryResponseObservationHandler().onStop(observationContext);
+
+ var otelSpan = TracingHelper.extractOtelSpan(tracingContext);
+ assertThat(otelSpan).isNotNull();
+ var spanData = ((ReadableSpan) otelSpan).toSpanData();
+ assertThat(spanData.getEvents().size()).isEqualTo(1);
+ assertThat(spanData.getEvents().get(0).getName())
+ .isEqualTo(VectorStoreObservationEventNames.CONTENT_QUERY_RESPONSE.value());
+ assertThat(spanData.getEvents()
+ .get(0)
+ .getAttributes()
+ .get(AttributeKey.stringArrayKey(VectorStoreObservationAttributes.DB_VECTOR_QUERY_CONTENT.value())))
+ .containsOnly("hello", "other-side");
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/AbstractVectorStoreBuilder.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/AbstractVectorStoreBuilder.java
new file mode 100644
index 000000000..0affbdcc8
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/AbstractVectorStoreBuilder.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore;
+
+import io.micrometer.observation.ObservationRegistry;
+
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention;
+import org.springframework.lang.Nullable;
+import org.springframework.util.Assert;
+
+/**
+ * Abstract base builder implementing common builder functionality for
+ * {@link VectorStore}. Provides default implementations for observation-related settings.
+ *
+ * @param the concrete builder type, enabling method chaining with the correct return
+ * type
+ */
+public abstract class AbstractVectorStoreBuilder>
+ implements VectorStore.Builder {
+
+ protected EmbeddingModel embeddingModel;
+
+ protected ObservationRegistry observationRegistry = ObservationRegistry.NOOP;
+
+ @Nullable
+ protected VectorStoreObservationConvention customObservationConvention;
+
+ public EmbeddingModel getEmbeddingModel() {
+ return this.embeddingModel;
+ }
+
+ public ObservationRegistry getObservationRegistry() {
+ return this.observationRegistry;
+ }
+
+ @Nullable
+ public VectorStoreObservationConvention getCustomObservationConvention() {
+ return this.customObservationConvention;
+ }
+
+ /**
+ * Returns this builder cast to the concrete builder type. Used internally to enable
+ * proper method chaining in subclasses.
+ * @return this builder cast to the concrete type
+ */
+ @SuppressWarnings("unchecked")
+ protected T self() {
+ return (T) this;
+ }
+
+ @Override
+ public T observationRegistry(ObservationRegistry observationRegistry) {
+ Assert.notNull(observationRegistry, "ObservationRegistry must not be null");
+ this.observationRegistry = observationRegistry;
+ return self();
+ }
+
+ @Override
+ public T customObservationConvention(VectorStoreObservationConvention convention) {
+ this.customObservationConvention = convention;
+ return self();
+ }
+
+ @Override
+ public T embeddingModel(EmbeddingModel embeddingModel) {
+ Assert.notNull(embeddingModel, "EmbeddingModel must not be null");
+ this.embeddingModel = embeddingModel;
+ return self();
+ }
+
+ protected void validate() {
+ Assert.notNull(this.embeddingModel, "EmbeddingModel must be configured");
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SearchRequest.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SearchRequest.java
new file mode 100644
index 000000000..597b39df0
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SearchRequest.java
@@ -0,0 +1,283 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore;
+
+import java.util.Objects;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.vectorstore.filter.Filter;
+import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
+import org.springframework.ai.vectorstore.filter.FilterExpressionTextParser;
+import org.springframework.lang.Nullable;
+import org.springframework.util.Assert;
+
+/**
+ * Similarity search request builder. Use the {@link #query(String)}, {@link #defaults()}
+ * or {@link #from(SearchRequest)} factory methods to create a new {@link SearchRequest}
+ * instance and then apply the 'with' methods to alter the default values.
+ *
+ * @author Christian Tzolov
+ * @author Thomas Vitale
+ */
+public final class SearchRequest {
+
+ /**
+ * Similarity threshold that accepts all search scores. A threshold value of 0.0 means
+ * any similarity is accepted or disable the similarity threshold filtering. A
+ * threshold value of 1.0 means an exact match is required.
+ */
+ public static final double SIMILARITY_THRESHOLD_ACCEPT_ALL = 0.0;
+
+ /**
+ * Default value for the top 'k' similar results to return.
+ */
+ public static final int DEFAULT_TOP_K = 4;
+
+ private String query;
+
+ private int topK = DEFAULT_TOP_K;
+
+ private double similarityThreshold = SIMILARITY_THRESHOLD_ACCEPT_ALL;
+
+ @Nullable
+ private Filter.Expression filterExpression;
+
+ private SearchRequest(String query) {
+ this.query = query;
+ }
+
+ /**
+ * Create a new {@link SearchRequest} builder instance with specified embedding query
+ * string.
+ * @param query Text to use for embedding similarity comparison.
+ * @return Returns new {@link SearchRequest} builder instance.
+ */
+ public static SearchRequest query(String query) {
+ Assert.notNull(query, "Query can not be null.");
+ return new SearchRequest(query);
+ }
+
+ /**
+ * Create a new {@link SearchRequest} builder instance with an empty embedding query
+ * string. Use the {@link #withQuery(String query)} to set/update the embedding query
+ * text.
+ * @return Returns new {@link SearchRequest} builder instance.
+ */
+ public static SearchRequest defaults() {
+ return new SearchRequest("");
+ }
+
+ /**
+ * Copy an existing {@link SearchRequest} instance.
+ * @param originalSearchRequest {@link SearchRequest} instance to copy.
+ * @return Returns new {@link SearchRequest} builder instance.
+ */
+ public static SearchRequest from(SearchRequest originalSearchRequest) {
+ return new SearchRequest(originalSearchRequest.getQuery()).withTopK(originalSearchRequest.getTopK())
+ .withSimilarityThreshold(originalSearchRequest.getSimilarityThreshold())
+ .withFilterExpression(originalSearchRequest.getFilterExpression());
+ }
+
+ /**
+ * @param query Text to use for embedding similarity comparison.
+ * @return this builder.
+ */
+ public SearchRequest withQuery(String query) {
+ Assert.notNull(query, "Query can not be null.");
+ this.query = query;
+ return this;
+ }
+
+ /**
+ * @param topK the top 'k' similar results to return.
+ * @return this builder.
+ */
+ public SearchRequest withTopK(int topK) {
+ Assert.isTrue(topK >= 0, "TopK should be positive.");
+ this.topK = topK;
+ return this;
+ }
+
+ /**
+ * Similarity threshold score to filter the search response by. Only documents with
+ * similarity score equal or greater than the 'threshold' will be returned. Note that
+ * this is a post-processing step performed on the client not the server side. A
+ * threshold value of 0.0 means any similarity is accepted or disable the similarity
+ * threshold filtering. A threshold value of 1.0 means an exact match is required.
+ * @param threshold The lower bound of the similarity score.
+ * @return this builder.
+ */
+ public SearchRequest withSimilarityThreshold(double threshold) {
+ Assert.isTrue(threshold >= 0 && threshold <= 1, "Similarity threshold must be in [0,1] range.");
+ this.similarityThreshold = threshold;
+ return this;
+ }
+
+ /**
+ * Sets disables the similarity threshold by setting it to 0.0 - all results are
+ * accepted.
+ * @return this builder.
+ */
+ public SearchRequest withSimilarityThresholdAll() {
+ return withSimilarityThreshold(SIMILARITY_THRESHOLD_ACCEPT_ALL);
+ }
+
+ /**
+ * Retrieves documents by query embedding similarity and matching the filters. Value
+ * of 'null' means that no metadata filters will be applied to the search.
+ *
+ * For example if the {@link Document#getMetadata()} schema is:
+ *
+ *
+ *
+ * you can constrain the search result to only UK countries with isActive=true and
+ * year equal or greater 2020. You can build this such metadata filter
+ * programmatically like this:
+ *
+ *
{@code
+ * var exp = new Filter.Expression(AND,
+ * new Expression(EQ, new Key("country"), new Value("UK")),
+ * new Expression(AND,
+ * new Expression(GTE, new Key("year"), new Value(2020)),
+ * new Expression(EQ, new Key("isActive"), new Value(true))));
+ * }
+ *
+ * The {@link Filter.Expression} is portable across all vector stores.
+ *
+ *
+ * The {@link FilterExpressionBuilder} is a DSL creating expressions programmatically:
+ *
+ *
{@code
+ * var b = new FilterExpressionBuilder();
+ * var exp = b.and(
+ * b.eq("country", "UK"),
+ * b.and(
+ * b.gte("year", 2020),
+ * b.eq("isActive", true)));
+ * }
+ *
+ * The {@link FilterExpressionTextParser} converts textual, SQL like filter expression
+ * language into {@link Filter.Expression}:
+ *
+ *
{@code
+ * var parser = new FilterExpressionTextParser();
+ * var exp = parser.parse("country == 'UK' && isActive == true && year >=2020");
+ * }
+ * @param expression {@link Filter.Expression} instance used to define the metadata
+ * filter criteria. The 'null' value stands for no expression filters.
+ * @return this builder.
+ */
+ public SearchRequest withFilterExpression(@Nullable Filter.Expression expression) {
+ this.filterExpression = expression;
+ return this;
+ }
+
+ /**
+ * Document metadata filter expression. For example if your
+ * {@link Document#getMetadata()} has a schema like:
+ *
+ *
+ *
+ * then you can constrain the search result with metadata filter expressions like:
+ *
+ *
{@code
+ * country == 'UK' && year >= 2020 && isActive == true
+ * Or
+ * country == 'BG' && (city NOT IN ['Sofia', 'Plovdiv'] || price < 134.34)
+ * }
+ *
+ * This ensures that the response contains only embeddings that match the specified
+ * filer criteria.
+ *
+ * The declarative, SQL like, filter syntax is portable across all vector stores
+ * supporting the filter search feature.
+ *
+ * The {@link FilterExpressionTextParser} is used to convert the text filter
+ * expression into {@link Filter.Expression}.
+ * @param textExpression declarative, portable, SQL like, metadata filter syntax. The
+ * 'null' value stands for no expression filters.
+ * @return this.builder
+ */
+ public SearchRequest withFilterExpression(@Nullable String textExpression) {
+ this.filterExpression = (textExpression != null) ? new FilterExpressionTextParser().parse(textExpression)
+ : null;
+ return this;
+ }
+
+ public String getQuery() {
+ return this.query;
+ }
+
+ public int getTopK() {
+ return this.topK;
+ }
+
+ public double getSimilarityThreshold() {
+ return this.similarityThreshold;
+ }
+
+ @Nullable
+ public Filter.Expression getFilterExpression() {
+ return this.filterExpression;
+ }
+
+ public boolean hasFilterExpression() {
+ return this.filterExpression != null;
+ }
+
+ @Override
+ public String toString() {
+ return "SearchRequest{" + "query='" + this.query + '\'' + ", topK=" + this.topK + ", similarityThreshold="
+ + this.similarityThreshold + ", filterExpression=" + this.filterExpression + '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SearchRequest that = (SearchRequest) o;
+ return this.topK == that.topK && Double.compare(that.similarityThreshold, this.similarityThreshold) == 0
+ && Objects.equals(this.query, that.query)
+ && Objects.equals(this.filterExpression, that.filterExpression);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(this.query, this.topK, this.similarityThreshold, this.filterExpression);
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java
new file mode 100644
index 000000000..28cbfc2eb
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java
@@ -0,0 +1,283 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import io.micrometer.observation.ObservationRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentMetadata;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.observation.conventions.VectorStoreProvider;
+import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric;
+import org.springframework.ai.util.JacksonUtils;
+import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention;
+import org.springframework.core.io.Resource;
+
+/**
+ * SimpleVectorStore is a simple implementation of the VectorStore interface.
+ *
+ * It also provides methods to save the current state of the vectors to a file, and to
+ * load vectors from a file.
+ *
+ * For a deeper understanding of the mathematical concepts and computations involved in
+ * calculating similarity scores among vectors, refer to this
+ * [resource](https://docs.spring.io/spring-ai/reference/api/vectordbs.html#_understanding_vectors).
+ *
+ * @author Raphael Yu
+ * @author Dingmeng Xue
+ * @author Mark Pollack
+ * @author Christian Tzolov
+ * @author Sebastien Deleuze
+ * @author Ilayaperumal Gopinathan
+ * @author Thomas Vitale
+ */
+public class SimpleVectorStore extends AbstractObservationVectorStore {
+
+ private static final Logger logger = LoggerFactory.getLogger(SimpleVectorStore.class);
+
+ private final ObjectMapper objectMapper;
+
+ protected Map store = new ConcurrentHashMap<>();
+
+ protected EmbeddingModel embeddingModel;
+
+ public SimpleVectorStore(EmbeddingModel embeddingModel) {
+ this(embeddingModel, ObservationRegistry.NOOP, null);
+ }
+
+ public SimpleVectorStore(EmbeddingModel embeddingModel, ObservationRegistry observationRegistry,
+ VectorStoreObservationConvention customObservationConvention) {
+
+ super(observationRegistry, customObservationConvention);
+
+ Objects.requireNonNull(embeddingModel, "EmbeddingModel must not be null");
+ this.embeddingModel = embeddingModel;
+ this.objectMapper = JsonMapper.builder().addModules(JacksonUtils.instantiateAvailableModules()).build();
+ }
+
+ @Override
+ public void doAdd(List documents) {
+ Objects.requireNonNull(documents, "Documents list cannot be null");
+ if (documents.isEmpty()) {
+ throw new IllegalArgumentException("Documents list cannot be empty");
+ }
+
+ for (Document document : documents) {
+ logger.info("Calling EmbeddingModel for document id = {}", document.getId());
+ float[] embedding = this.embeddingModel.embed(document);
+ SimpleVectorStoreContent storeContent = new SimpleVectorStoreContent(document.getId(),
+ document.getContent(), document.getMetadata(), embedding);
+ this.store.put(document.getId(), storeContent);
+ }
+ }
+
+ @Override
+ public Optional doDelete(List idList) {
+ for (String id : idList) {
+ this.store.remove(id);
+ }
+ return Optional.of(true);
+ }
+
+ @Override
+ public List doSimilaritySearch(SearchRequest request) {
+ if (request.getFilterExpression() != null) {
+ throw new UnsupportedOperationException(
+ "The [" + this.getClass() + "] doesn't support metadata filtering!");
+ }
+
+ float[] userQueryEmbedding = getUserQueryEmbedding(request.getQuery());
+ return this.store.values()
+ .stream()
+ .map(content -> content
+ .toDocument(EmbeddingMath.cosineSimilarity(userQueryEmbedding, content.getEmbedding())))
+ .filter(document -> document.getScore() >= request.getSimilarityThreshold())
+ .sorted(Comparator.comparing(Document::getScore).reversed())
+ .limit(request.getTopK())
+ .toList();
+ }
+
+ /**
+ * Serialize the vector store content into a file in JSON format.
+ * @param file the file to save the vector store content
+ */
+ public void save(File file) {
+ String json = getVectorDbAsJson();
+ try {
+ if (!file.exists()) {
+ logger.info("Creating new vector store file: {}", file);
+ try {
+ Files.createFile(file.toPath());
+ }
+ catch (FileAlreadyExistsException e) {
+ throw new RuntimeException("File already exists: " + file, e);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Failed to create new file: " + file + ". Reason: " + e.getMessage(), e);
+ }
+ }
+ else {
+ logger.info("Overwriting existing vector store file: {}", file);
+ }
+ try (OutputStream stream = new FileOutputStream(file);
+ Writer writer = new OutputStreamWriter(stream, StandardCharsets.UTF_8)) {
+ writer.write(json);
+ writer.flush();
+ }
+ }
+ catch (IOException ex) {
+ logger.error("IOException occurred while saving vector store file.", ex);
+ throw new RuntimeException(ex);
+ }
+ catch (SecurityException ex) {
+ logger.error("SecurityException occurred while saving vector store file.", ex);
+ throw new RuntimeException(ex);
+ }
+ catch (NullPointerException ex) {
+ logger.error("NullPointerException occurred while saving vector store file.", ex);
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ * Deserialize the vector store content from a file in JSON format into memory.
+ * @param file the file to load the vector store content
+ */
+ public void load(File file) {
+ TypeReference> typeRef = new TypeReference<>() {
+
+ };
+ try {
+ this.store = this.objectMapper.readValue(file, typeRef);
+ }
+ catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ * Deserialize the vector store content from a resource in JSON format into memory.
+ * @param resource the resource to load the vector store content
+ */
+ public void load(Resource resource) {
+ TypeReference> typeRef = new TypeReference<>() {
+
+ };
+ try {
+ this.store = this.objectMapper.readValue(resource.getInputStream(), typeRef);
+ }
+ catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private String getVectorDbAsJson() {
+ ObjectWriter objectWriter = this.objectMapper.writerWithDefaultPrettyPrinter();
+ String json;
+ try {
+ json = objectWriter.writeValueAsString(this.store);
+ }
+ catch (JsonProcessingException e) {
+ throw new RuntimeException("Error serializing documentMap to JSON.", e);
+ }
+ return json;
+ }
+
+ private float[] getUserQueryEmbedding(String query) {
+ return this.embeddingModel.embed(query);
+ }
+
+ @Override
+ public VectorStoreObservationContext.Builder createObservationContextBuilder(String operationName) {
+
+ return VectorStoreObservationContext.builder(VectorStoreProvider.SIMPLE.value(), operationName)
+ .withDimensions(this.embeddingModel.dimensions())
+ .withCollectionName("in-memory-map")
+ .withSimilarityMetric(VectorStoreSimilarityMetric.COSINE.value());
+ }
+
+ public static final class EmbeddingMath {
+
+ private EmbeddingMath() {
+ throw new UnsupportedOperationException("This is a utility class and cannot be instantiated");
+ }
+
+ public static double cosineSimilarity(float[] vectorX, float[] vectorY) {
+ if (vectorX == null || vectorY == null) {
+ throw new RuntimeException("Vectors must not be null");
+ }
+ if (vectorX.length != vectorY.length) {
+ throw new IllegalArgumentException("Vectors lengths must be equal");
+ }
+
+ float dotProduct = dotProduct(vectorX, vectorY);
+ float normX = norm(vectorX);
+ float normY = norm(vectorY);
+
+ if (normX == 0 || normY == 0) {
+ throw new IllegalArgumentException("Vectors cannot have zero norm");
+ }
+
+ return dotProduct / (Math.sqrt(normX) * Math.sqrt(normY));
+ }
+
+ public static float dotProduct(float[] vectorX, float[] vectorY) {
+ if (vectorX.length != vectorY.length) {
+ throw new IllegalArgumentException("Vectors lengths must be equal");
+ }
+
+ float result = 0;
+ for (int i = 0; i < vectorX.length; ++i) {
+ result += vectorX[i] * vectorY[i];
+ }
+
+ return result;
+ }
+
+ public static float norm(float[] vector) {
+ return dotProduct(vector, vector);
+ }
+
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStoreContent.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStoreContent.java
new file mode 100644
index 000000000..b44389678
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStoreContent.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentMetadata;
+import org.springframework.ai.document.id.IdGenerator;
+import org.springframework.ai.document.id.RandomIdGenerator;
+import org.springframework.ai.model.Content;
+import org.springframework.util.Assert;
+
+/**
+ * An immutable {@link Content} implementation representing content, metadata, and its
+ * embeddings. This class is thread-safe and all its fields are final and deeply
+ * immutable. The embedding vector is required for all instances of this class.
+ */
+public final class SimpleVectorStoreContent implements Content {
+
+ private final String id;
+
+ private final String text;
+
+ private final Map metadata;
+
+ private final float[] embedding;
+
+ /**
+ * Creates a new instance with the given content, empty metadata, and embedding
+ * vector.
+ * @param text the content text, must not be null
+ * @param embedding the embedding vector, must not be null
+ */
+ @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
+ public SimpleVectorStoreContent(@JsonProperty("text") @JsonAlias({ "content" }) String text,
+ @JsonProperty("embedding") float[] embedding) {
+ this(text, new HashMap<>(), embedding);
+ }
+
+ /**
+ * Creates a new instance with the given content, metadata, and embedding vector.
+ * @param text the content text, must not be null
+ * @param metadata the metadata map, must not be null
+ * @param embedding the embedding vector, must not be null
+ */
+ public SimpleVectorStoreContent(String text, Map metadata, float[] embedding) {
+ this(text, metadata, new RandomIdGenerator(), embedding);
+ }
+
+ /**
+ * Creates a new instance with the given content, metadata, custom ID generator, and
+ * embedding vector.
+ * @param text the content text, must not be null
+ * @param metadata the metadata map, must not be null
+ * @param idGenerator the ID generator to use, must not be null
+ * @param embedding the embedding vector, must not be null
+ */
+ public SimpleVectorStoreContent(String text, Map metadata, IdGenerator idGenerator,
+ float[] embedding) {
+ this(idGenerator.generateId(text, metadata), text, metadata, embedding);
+ }
+
+ /**
+ * Creates a new instance with all fields specified.
+ * @param id the unique identifier, must not be empty
+ * @param text the content text, must not be null
+ * @param metadata the metadata map, must not be null
+ * @param embedding the embedding vector, must not be null
+ * @throws IllegalArgumentException if any parameter is null or if id is empty
+ */
+ public SimpleVectorStoreContent(String id, String text, Map metadata, float[] embedding) {
+ Assert.hasText(id, "id must not be null or empty");
+ Assert.notNull(text, "content must not be null");
+ Assert.notNull(metadata, "metadata must not be null");
+ Assert.notNull(embedding, "embedding must not be null");
+ Assert.isTrue(embedding.length > 0, "embedding vector must not be empty");
+
+ this.id = id;
+ this.text = text;
+ this.metadata = Collections.unmodifiableMap(new HashMap<>(metadata));
+ this.embedding = Arrays.copyOf(embedding, embedding.length);
+ }
+
+ /**
+ * Creates a new instance with an updated embedding vector.
+ * @param embedding the new embedding vector, must not be null
+ * @return a new instance with the updated embedding
+ * @throws IllegalArgumentException if embedding is null or empty
+ */
+ public SimpleVectorStoreContent withEmbedding(float[] embedding) {
+ Assert.notNull(embedding, "embedding must not be null");
+ Assert.isTrue(embedding.length > 0, "embedding vector must not be empty");
+ return new SimpleVectorStoreContent(this.id, this.text, this.metadata, embedding);
+ }
+
+ public String getId() {
+ return this.id;
+ }
+
+ @Override
+ public String getText() {
+ return this.text;
+ }
+
+ @Override
+ public String getContent() {
+ return this.text;
+ }
+
+ @Override
+ public Map getMetadata() {
+ return this.metadata;
+ }
+
+ /**
+ * Returns a defensive copy of the embedding vector.
+ * @return a new array containing the embedding vector
+ */
+ public float[] getEmbedding() {
+ return Arrays.copyOf(this.embedding, this.embedding.length);
+ }
+
+ public Document toDocument(Double score) {
+ var metadata = new HashMap<>(this.metadata);
+ metadata.put(DocumentMetadata.DISTANCE.value(), 1.0 - score);
+ return Document.builder().id(this.id).text(this.text).metadata(metadata).score(score).build();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SimpleVectorStoreContent that = (SimpleVectorStoreContent) o;
+ return Objects.equals(this.id, that.id) && Objects.equals(this.text, that.text)
+ && Objects.equals(this.metadata, that.metadata) && Arrays.equals(this.embedding, that.embedding);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hashCode(this.id);
+ result = 31 * result + Objects.hashCode(this.text);
+ result = 31 * result + Objects.hashCode(this.metadata);
+ result = 31 * result + Arrays.hashCode(this.embedding);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "SimpleVectorStoreContent{" + "id='" + this.id + '\'' + ", content='" + this.text + '\'' + ", metadata="
+ + this.metadata + ", embedding=" + Arrays.toString(this.embedding) + '}';
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/VectorStore.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/VectorStore.java
new file mode 100644
index 000000000..faea243a7
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/VectorStore.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore;
+
+import java.util.List;
+import java.util.Optional;
+
+import io.micrometer.observation.ObservationRegistry;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentWriter;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.vectorstore.observation.DefaultVectorStoreObservationConvention;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention;
+import org.springframework.lang.Nullable;
+
+/**
+ * The {@code VectorStore} interface defines the operations for managing and querying
+ * documents in a vector database. It extends {@link DocumentWriter} to support document
+ * writing operations. Vector databases are specialized for AI applications, performing
+ * similarity searches based on vector representations of data rather than exact matches.
+ * This interface allows for adding, deleting, and searching documents based on their
+ * similarity to a given query.
+ */
+public interface VectorStore extends DocumentWriter {
+
+ default String getName() {
+ return this.getClass().getSimpleName();
+ }
+
+ /**
+ * Adds list of {@link Document}s to the vector store.
+ * @param documents the list of documents to store. Throws an exception if the
+ * underlying provider checks for duplicate IDs.
+ */
+ void add(List documents);
+
+ @Override
+ default void accept(List documents) {
+ add(documents);
+ }
+
+ /**
+ * Deletes documents from the vector store.
+ * @param idList list of document ids for which documents will be removed.
+ * @return Returns true if the documents were successfully deleted.
+ */
+ Optional delete(List idList);
+
+ /**
+ * Retrieves documents by query embedding similarity and metadata filters to retrieve
+ * exactly the number of nearest-neighbor results that match the request criteria.
+ * @param request Search request for set search parameters, such as the query text,
+ * topK, similarity threshold and metadata filter expressions.
+ * @return Returns documents th match the query request conditions.
+ */
+ List similaritySearch(SearchRequest request);
+
+ /**
+ * Retrieves documents by query embedding similarity using the default
+ * {@link SearchRequest}'s' search criteria.
+ * @param query Text to use for embedding similarity comparison.
+ * @return Returns a list of documents that have embeddings similar to the query text
+ * embedding.
+ */
+ default List similaritySearch(String query) {
+ return this.similaritySearch(SearchRequest.query(query));
+ }
+
+ /**
+ * Builder interface for creating VectorStore instances. Implements a fluent builder
+ * pattern for configuring observation-related settings.
+ *
+ * @param the concrete builder type, enabling method chaining with the correct
+ * return type
+ */
+ interface Builder> {
+
+ T embeddingModel(EmbeddingModel embeddingModel);
+
+ /**
+ * Sets the registry for collecting observations and metrics. Defaults to
+ * {@link ObservationRegistry#NOOP} if not specified.
+ * @param observationRegistry the registry to use for observations
+ * @return the builder instance for method chaining
+ */
+ T observationRegistry(ObservationRegistry observationRegistry);
+
+ /**
+ * Sets a custom convention for creating observations. If not specified,
+ * {@link DefaultVectorStoreObservationConvention} will be used.
+ * @param convention the custom observation convention to use
+ * @return the builder instance for method chaining
+ */
+ T customObservationConvention(VectorStoreObservationConvention convention);
+
+ /**
+ * Builds and returns a new VectorStore instance with the configured settings.
+ * @return a new VectorStore instance
+ */
+ VectorStore build();
+
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/Filter.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/Filter.java
new file mode 100644
index 000000000..53e16c691
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/Filter.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.filter;
+
+/**
+ * Portable runtime generative for metadata filter expressions. This generic generative is
+ * used to define store agnostic filter expressions than later can be converted into
+ * vector-store specific, native, expressions.
+ *
+ * The expression generative supports constant comparison
+ * {@code (e.g. ==, !=, <, <=, >, >=) }, IN/NON-IN checks and AND and OR to compose
+ * multiple expressions.
+ *
+ * For example:
+ *
+ *
{@code
+ * // 1: country == "BG"
+ * new Expression(EQ, new Key("country"), new Value("BG"));
+ *
+ * // 2: genre == "drama" AND year >= 2020
+ * new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")),
+ * new Expression(GTE, new Key("year"), new Value(2020)));
+ *
+ * // 3: genre in ["comedy", "documentary", "drama"]
+ * new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama")));
+ *
+ * // 4: year >= 2020 OR country == "BG" AND city != "Sofia"
+ * new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)),
+ * new Expression(AND, new Expression(EQ, new Key("country"), new Value("BG")),
+ * new Expression(NE, new Key("city"), new Value("Sofia"))));
+ *
+ * // 5: (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"]
+ * new Expression(AND,
+ * new Group(new Expression(OR, new Expression(EQ, new Key("country"), new Value("BG")),
+ * new Expression(GTE, new Key("year"), new Value(2020)))),
+ * new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Varna"))));
+ *
+ * // 6: isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]
+ * new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)),
+ * new Expression(AND, new Expression(GTE, new Key("year"), new Value(2020)),
+ * new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))));
+ *
+ * }
+ *
+ *
+ * Usually you will not create expression manually but use either the
+ * {@link FilterExpressionBuilder} DSL or the {@link FilterExpressionTextParser} for
+ * parsing generic text expressions.
+ *
+ * @author Christian Tzolov
+ */
+public class Filter {
+
+ /**
+ * Filter expression operations.
+ *
+ * - EQ, NE, GT, GTE, LT, LTE operations supports "Key ExprType Value"
+ * expressions.
+ *
+ * - AND, OR are binary operations that support "(Expression|Group) ExprType
+ * (Expression|Group)" expressions.
+ *
+ * - IN, NIN support "Key (IN|NIN) ArrayValue" expression.
+ */
+ public enum ExpressionType {
+
+ AND, OR, EQ, NE, GT, GTE, LT, LTE, IN, NIN, NOT
+
+ }
+
+ /**
+ * Mark interface representing the supported expression types: {@link Key},
+ * {@link Value}, {@link Expression} and {@link Group}.
+ */
+ public interface Operand {
+
+ }
+
+ /**
+ * String identifier representing an expression key. (e.g. the country in the country
+ * == "NL" expression).
+ *
+ * @param key expression key
+ */
+ public record Key(String key) implements Operand {
+
+ }
+
+ /**
+ * Represents expression value constant or constant array. Support Numeric, Boolean
+ * and String data types.
+ *
+ * @param value value constant or constant array
+ */
+ public record Value(Object value) implements Operand {
+
+ }
+
+ /**
+ * Triple that represents and filter boolean expression as
+ * left type right.
+ *
+ * @param type Specify the expression type.
+ * @param left For comparison and inclusion expression types, the operand must be of
+ * type {@link Key} and for the AND|OR expression types the left operand must be
+ * another {@link Expression}.
+ * @param right For comparison and inclusion expression types, the operand must be of
+ * type {@link Value} or array of values. For the AND|OR type the right operand must
+ * be another {@link Expression}.
+ */
+ public record Expression(ExpressionType type, Operand left, Operand right) implements Operand {
+
+ public Expression(ExpressionType type, Operand operand) {
+ this(type, operand, null);
+ }
+
+ }
+
+ /**
+ * Represents expression grouping (e.g. (...) ) that indicates that the group needs to
+ * be evaluated with a precedence.
+ *
+ * @param content Inner expression to be evaluated as a part of the group.
+ */
+ public record Group(Expression content) implements Operand {
+
+ }
+
+}
diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilder.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilder.java
new file mode 100644
index 000000000..f7410c898
--- /dev/null
+++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilder.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.filter;
+
+import java.util.List;
+
+import org.springframework.ai.vectorstore.filter.Filter.ExpressionType;
+import org.springframework.ai.vectorstore.filter.Filter.Key;
+import org.springframework.ai.vectorstore.filter.Filter.Value;
+
+/**
+ * DSL builder for {@link Filter.Expression} instances. Here are some common examples:
+ *
+ *
{@code
+ * var b = new FilterExpressionBuilder();
+ *
+ * // 1: country == "BG"
+ * var exp1 = b.eq("country", "BG");
+ *
+ * // 2: genre == "drama" AND year >= 2020
+ * var exp2 = b.and(b.eq("genre", "drama"), b.gte("year", 2020));
+ *
+ * // 3: genre in ["comedy", "documentary", "drama"]
+ * var exp3 = b.in("genre", "comedy", "documentary", "drama");
+ *
+ * // 4: year >= 2020 OR country == "BG" AND city != "Sofia"
+ * var exp4 = b.and(b.or(b.gte("year", 2020), b.eq("country", "BG")), b.ne("city", "Sofia"));
+ *
+ * // 5: (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"]
+ * var exp5 = b.and(b.group(b.or(b.gte("year", 2020), b.eq("country", "BG"))), b.nin("city", "Sofia", "Plovdiv"));
+ *
+ * // 6: isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]
+ * var exp6 = b.and(b.and(b.eq("isOpen", true), b.gte("year", 2020)), b.in("country", "BG", "NL", "US"));
+ *
+ * }
+ *
+ *
+ * This builder DSL mimics the common https://www.baeldung.com/hibernate-criteria-queries
+ * syntax.
+ *
+ * @author Christian Tzolov
+ */
+public class FilterExpressionBuilder {
+
+ public Op eq(String key, Object value) {
+ return new Op(new Filter.Expression(ExpressionType.EQ, new Key(key), new Value(value)));
+ }
+
+ public Op ne(String key, Object value) {
+ return new Op(new Filter.Expression(ExpressionType.NE, new Key(key), new Value(value)));
+ }
+
+ public Op gt(String key, Object value) {
+ return new Op(new Filter.Expression(ExpressionType.GT, new Key(key), new Value(value)));
+ }
+
+ public Op gte(String key, Object value) {
+ return new Op(new Filter.Expression(ExpressionType.GTE, new Key(key), new Value(value)));
+ }
+
+ public Op lt(String key, Object value) {
+ return new Op(new Filter.Expression(ExpressionType.LT, new Key(key), new Value(value)));
+ }
+
+ public Op lte(String key, Object value) {
+ return new Op(new Filter.Expression(ExpressionType.LTE, new Key(key), new Value(value)));
+ }
+
+ public Op and(Op left, Op right) {
+ return new Op(new Filter.Expression(ExpressionType.AND, left.expression, right.expression));
+ }
+
+ public Op or(Op left, Op right) {
+ return new Op(new Filter.Expression(ExpressionType.OR, left.expression, right.expression));
+ }
+
+ public Op in(String key, Object... values) {
+ return this.in(key, List.of(values));
+ }
+
+ public Op in(String key, List