From bff28299a2cdbff18a7f60cffd6e43937616cd4d Mon Sep 17 00:00:00 2001 From: Mark Pollack Date: Wed, 18 Dec 2024 14:12:37 -0500 Subject: [PATCH] wip --- pom.xml | 1 + spring-ai-vector-store/pom.xml | 100 ++ .../SimpleVectorStoreSimilarityTests.java | 49 + .../vectorstore/SimpleVectorStoreTests.java | 260 +++ .../filter/FilterExpressionBuilderTests.java | 124 ++ .../FilterExpressionTextParserTests.java | 201 +++ .../vectorstore/filter/FilterHelperTests.java | 171 ++ .../filter/SearchRequestTests.java | 139 ++ ...ineconeFilterExpressionConverterTests.java | 126 ++ ...VectorStoreObservationConventionTests.java | 130 ++ .../VectorStoreObservationContextTests.java | 53 + ...reQueryResponseObservationFilterTests.java | 73 + ...eQueryResponseObservationHandlerTests.java | 71 + .../AbstractVectorStoreBuilder.java | 90 ++ .../ai/vectorstore/SearchRequest.java | 283 ++++ .../ai/vectorstore/SimpleVectorStore.java | 283 ++++ .../vectorstore/SimpleVectorStoreContent.java | 180 +++ .../ai/vectorstore/VectorStore.java | 119 ++ .../ai/vectorstore/filter/Filter.java | 143 ++ .../filter/FilterExpressionBuilder.java | 130 ++ .../filter/FilterExpressionConverter.java | 35 + .../filter/FilterExpressionTextParser.java | 303 ++++ .../ai/vectorstore/filter/FilterHelper.java | 204 +++ .../vectorstore/filter/antlr4/Filters.interp | 69 + .../filter/antlr4/FiltersBaseListener.java | 411 +++++ .../filter/antlr4/FiltersBaseVisitor.java | 233 +++ .../filter/antlr4/FiltersLexer.interp | 98 ++ .../filter/antlr4/FiltersLexer.java | 311 ++++ .../filter/antlr4/FiltersListener.java | 235 +++ .../filter/antlr4/FiltersParser.java | 1406 +++++++++++++++++ .../filter/antlr4/FiltersVisitor.java | 152 ++ .../AbstractFilterExpressionConverter.java | 229 +++ .../PineconeFilterExpressionConverter.java | 64 + .../PrintFilterExpressionConverter.java | 51 + .../AbstractObservationVectorStore.java | 154 ++ ...faultVectorStoreObservationConvention.java | 161 ++ ...ectorStoreObservationContentProcessor.java | 42 + .../VectorStoreObservationContext.java | 228 +++ .../VectorStoreObservationConvention.java | 36 + .../VectorStoreObservationDocumentation.java | 208 +++ ...orStoreQueryResponseObservationFilter.java | 53 + ...rStoreQueryResponseObservationHandler.java | 59 + .../vectorstore/observation/package-info.java | 25 + .../ai/vectorstore/package-info.java | 22 + .../ai/vectorstore/filter/antlr4/Filters.g4 | 0 45 files changed, 7515 insertions(+) create mode 100644 spring-ai-vector-store/pom.xml create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreSimilarityTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilderTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterHelperTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/SearchRequestTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverterTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConventionTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContextTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilterTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandlerTests.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/AbstractVectorStoreBuilder.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SearchRequest.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStoreContent.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/VectorStore.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/Filter.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilder.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionConverter.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParser.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterHelper.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/Filters.interp create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersBaseListener.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersBaseVisitor.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersLexer.interp create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersLexer.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersListener.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersParser.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersVisitor.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/AbstractFilterExpressionConverter.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverter.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/PrintFilterExpressionConverter.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/AbstractObservationVectorStore.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConvention.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContentProcessor.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContext.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationConvention.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationDocumentation.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilter.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandler.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/package-info.java create mode 100644 spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/package-info.java rename {spring-ai-core => spring-ai-vector-store}/src/main/resources/antlr4/org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 (100%) diff --git a/pom.xml b/pom.xml index 767db5655..2198a8da0 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ spring-ai-docs spring-ai-bom spring-ai-core + spring-ai-vector-store spring-ai-test spring-ai-spring-boot-autoconfigure spring-ai-retry diff --git a/spring-ai-vector-store/pom.xml b/spring-ai-vector-store/pom.xml new file mode 100644 index 000000000..ddad91a63 --- /dev/null +++ b/spring-ai-vector-store/pom.xml @@ -0,0 +1,100 @@ + + + + + 4.0.0 + + org.springframework.ai + spring-ai + 1.0.0-SNAPSHOT + + spring-ai-vector-store + jar + Spring AI Vector Store + Spring AI Vector Store APIs + https://github.com/spring-projects/spring-ai + + + https://github.com/spring-projects/spring-ai + git://github.com/spring-projects/spring-ai.git + git@github.com:spring-projects/spring-ai.git + + + + + + + + + + io.micrometer + micrometer-core + + + + io.micrometer + context-propagation + + + + io.micrometer + micrometer-tracing-bridge-otel + true + + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + antlr4 + + false + + + + + org.antlr + antlr4-maven-plugin + ${antlr.version} + + ${basedir}/src/main/resources/antlr4 + ${basedir}/src/main/java + + true + + + + + antlr4 + + + + + + + + + diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreSimilarityTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreSimilarityTests.java new file mode 100644 index 000000000..499d0269a --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreSimilarityTests.java @@ -0,0 +1,49 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +import org.springframework.ai.document.Document; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Ilayaperumal Gopinathan + * @author Thomas Vitale + */ +public class SimpleVectorStoreSimilarityTests { + + @Test + public void testSimilarity() { + Map metadata = new HashMap<>(); + metadata.put("foo", "bar"); + float[] testEmbedding = new float[] { 1.0f, 2.0f, 3.0f }; + + SimpleVectorStoreContent storeContent = new SimpleVectorStoreContent("1", "hello, how are you?", metadata, + testEmbedding); + Document document = storeContent.toDocument(0.6); + assertThat(document).isNotNull(); + assertThat(document.getId()).isEqualTo("1"); + assertThat(document.getContent()).isEqualTo("hello, how are you?"); + assertThat(document.getMetadata().get("foo")).isEqualTo("bar"); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreTests.java new file mode 100644 index 000000000..4e81eb5d3 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreTests.java @@ -0,0 +1,260 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.CleanupMode; +import org.junit.jupiter.api.io.TempDir; + +import org.springframework.ai.document.Document; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.core.io.Resource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class SimpleVectorStoreTests { + + @TempDir(cleanup = CleanupMode.ON_SUCCESS) + Path tempDir; + + private SimpleVectorStore vectorStore; + + private EmbeddingModel mockEmbeddingModel; + + @BeforeEach + void setUp() { + this.mockEmbeddingModel = mock(EmbeddingModel.class); + when(this.mockEmbeddingModel.dimensions()).thenReturn(3); + when(this.mockEmbeddingModel.embed(any(String.class))).thenReturn(new float[] { 0.1f, 0.2f, 0.3f }); + when(this.mockEmbeddingModel.embed(any(Document.class))).thenReturn(new float[] { 0.1f, 0.2f, 0.3f }); + this.vectorStore = new SimpleVectorStore(this.mockEmbeddingModel); + } + + @Test + void shouldAddAndRetrieveDocument() { + Document doc = Document.builder().id("1").text("test content").metadata(Map.of("key", "value")).build(); + + this.vectorStore.add(List.of(doc)); + + List results = this.vectorStore.similaritySearch("test content"); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("test content"); + assertThat(result.getMetadata()).containsEntry("key", "value"); + }); + } + + @Test + void shouldAddMultipleDocuments() { + List docs = Arrays.asList(Document.builder().id("1").text("first").build(), + Document.builder().id("2").text("second").build()); + + this.vectorStore.add(docs); + + List results = this.vectorStore.similaritySearch("first"); + assertThat(results).hasSize(2).extracting(Document::getId).containsExactlyInAnyOrder("1", "2"); + } + + @Test + void shouldHandleEmptyDocumentList() { + assertThatThrownBy(() -> this.vectorStore.add(Collections.emptyList())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Documents list cannot be empty"); + } + + @Test + void shouldHandleNullDocumentList() { + assertThatThrownBy(() -> this.vectorStore.add(null)).isInstanceOf(NullPointerException.class) + .hasMessage("Documents list cannot be null"); + } + + @Test + void shouldDeleteDocuments() { + Document doc = Document.builder().id("1").text("test content").build(); + + this.vectorStore.add(List.of(doc)); + assertThat(this.vectorStore.similaritySearch("test")).hasSize(1); + + this.vectorStore.delete(List.of("1")); + assertThat(this.vectorStore.similaritySearch("test")).isEmpty(); + } + + @Test + void shouldHandleDeleteOfNonexistentDocument() { + this.vectorStore.delete(List.of("nonexistent-id")); + // Should not throw exception and return true + assertThat(this.vectorStore.delete(List.of("nonexistent-id")).get()).isTrue(); + } + + @Test + void shouldPerformSimilaritySearchWithThreshold() { + // Configure mock to return different embeddings for different queries + when(this.mockEmbeddingModel.embed("query")).thenReturn(new float[] { 0.9f, 0.9f, 0.9f }); + + Document doc = Document.builder().id("1").text("test content").build(); + + this.vectorStore.add(List.of(doc)); + + SearchRequest request = SearchRequest.query("query").withSimilarityThreshold(0.99f).withTopK(5); + + List results = this.vectorStore.similaritySearch(request); + assertThat(results).isEmpty(); + } + + @Test + void shouldSaveAndLoadVectorStore() throws IOException { + Document doc = Document.builder() + .id("1") + .text("test content") + .metadata(new HashMap<>(Map.of("key", "value"))) + .build(); + + this.vectorStore.add(List.of(doc)); + + File saveFile = this.tempDir.resolve("vector-store.json").toFile(); + this.vectorStore.save(saveFile); + + SimpleVectorStore loadedStore = new SimpleVectorStore(this.mockEmbeddingModel); + loadedStore.load(saveFile); + + List results = loadedStore.similaritySearch("test content"); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("test content"); + assertThat(result.getMetadata()).containsEntry("key", "value"); + }); + } + + @Test + void shouldHandleLoadFromInvalidResource() throws IOException { + Resource mockResource = mock(Resource.class); + when(mockResource.getInputStream()).thenThrow(new IOException("Resource not found")); + + assertThatThrownBy(() -> this.vectorStore.load(mockResource)).isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(IOException.class) + .hasMessageContaining("Resource not found"); + } + + @Test + void shouldHandleSaveToInvalidLocation() { + File invalidFile = new File("/invalid/path/file.json"); + + assertThatThrownBy(() -> this.vectorStore.save(invalidFile)).isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(IOException.class); + } + + @Test + void shouldHandleConcurrentOperations() throws InterruptedException { + int numThreads = 10; + Thread[] threads = new Thread[numThreads]; + + for (int i = 0; i < numThreads; i++) { + final String id = String.valueOf(i); + threads[i] = new Thread(() -> { + Document doc = Document.builder().id(id).text("content " + id).build(); + this.vectorStore.add(List.of(doc)); + }); + threads[i].start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + SearchRequest request = SearchRequest.query("test").withTopK(numThreads); + + List results = this.vectorStore.similaritySearch(request); + + assertThat(results).hasSize(numThreads); + + // Verify all documents were properly added + Set resultIds = results.stream().map(Document::getId).collect(Collectors.toSet()); + + Set expectedIds = new java.util.HashSet<>(); + for (int i = 0; i < numThreads; i++) { + expectedIds.add(String.valueOf(i)); + } + + assertThat(resultIds).containsExactlyInAnyOrderElementsOf(expectedIds); + + // Verify content integrity + results.forEach(doc -> assertThat(doc.getContent()).isEqualTo("content " + doc.getId())); + } + + @Test + void shouldRejectInvalidSimilarityThreshold() { + assertThatThrownBy(() -> SearchRequest.query("test").withSimilarityThreshold(2.0f)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Similarity threshold must be in [0,1] range."); + } + + @Test + void shouldRejectNegativeTopK() { + assertThatThrownBy(() -> SearchRequest.query("test").withTopK(-1)).isInstanceOf(IllegalArgumentException.class) + .hasMessage("TopK should be positive."); + } + + @Test + void shouldHandleCosineSimilarityEdgeCases() { + float[] zeroVector = new float[] { 0f, 0f, 0f }; + float[] normalVector = new float[] { 1f, 1f, 1f }; + + assertThatThrownBy(() -> SimpleVectorStore.EmbeddingMath.cosineSimilarity(zeroVector, normalVector)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Vectors cannot have zero norm"); + } + + @Test + void shouldHandleVectorLengthMismatch() { + float[] vector1 = new float[] { 1f, 2f }; + float[] vector2 = new float[] { 1f, 2f, 3f }; + + assertThatThrownBy(() -> SimpleVectorStore.EmbeddingMath.cosineSimilarity(vector1, vector2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Vectors lengths must be equal"); + } + + @Test + void shouldHandleNullVectors() { + float[] vector = new float[] { 1f, 2f, 3f }; + + assertThatThrownBy(() -> SimpleVectorStore.EmbeddingMath.cosineSimilarity(null, vector)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Vectors must not be null"); + + assertThatThrownBy(() -> SimpleVectorStore.EmbeddingMath.cosineSimilarity(vector, null)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Vectors must not be null"); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilderTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilderTests.java new file mode 100644 index 000000000..12084d007 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilderTests.java @@ -0,0 +1,124 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.springframework.ai.vectorstore.filter.Filter.Expression; +import org.springframework.ai.vectorstore.filter.Filter.Group; +import org.springframework.ai.vectorstore.filter.Filter.Key; +import org.springframework.ai.vectorstore.filter.Filter.Value; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.AND; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.EQ; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.GTE; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.IN; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NE; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NIN; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NOT; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.OR; + +/** + * @author Christian Tzolov + */ +public class FilterExpressionBuilderTests { + + FilterExpressionBuilder b = new FilterExpressionBuilder(); + + @Test + public void testEQ() { + // country == "BG" + assertThat(this.b.eq("country", "BG").build()) + .isEqualTo(new Expression(EQ, new Key("country"), new Value("BG"))); + } + + @Test + public void tesEqAndGte() { + // genre == "drama" AND year >= 2020 + Expression exp = this.b.and(this.b.eq("genre", "drama"), this.b.gte("year", 2020)).build(); + assertThat(exp).isEqualTo(new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")), + new Expression(GTE, new Key("year"), new Value(2020)))); + } + + @Test + public void testIn() { + // genre in ["comedy", "documentary", "drama"] + var exp = this.b.in("genre", "comedy", "documentary", "drama").build(); + assertThat(exp) + .isEqualTo(new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama")))); + } + + @Test + public void testNe() { + // year >= 2020 OR country == "BG" AND city != "Sofia" + var exp = this.b + .and(this.b.or(this.b.gte("year", 2020), this.b.eq("country", "BG")), this.b.ne("city", "Sofia")) + .build(); + + assertThat(exp).isEqualTo(new Expression(AND, + new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)), + new Expression(EQ, new Key("country"), new Value("BG"))), + new Expression(NE, new Key("city"), new Value("Sofia")))); + } + + @Test + public void testGroup() { + // (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"] + var exp = this.b + .and(this.b.group(this.b.or(this.b.gte("year", 2020), this.b.eq("country", "BG"))), + this.b.nin("city", "Sofia", "Plovdiv")) + .build(); + + assertThat(exp).isEqualTo(new Expression(AND, + new Group(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)), + new Expression(EQ, new Key("country"), new Value("BG")))), + new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Plovdiv"))))); + } + + @Test + public void tesIn2() { + // isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"] + var exp = this.b + .and(this.b.and(this.b.eq("isOpen", true), this.b.gte("year", 2020)), + this.b.in("country", "BG", "NL", "US")) + .build(); + + assertThat(exp).isEqualTo(new Expression(AND, + new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)), + new Expression(GTE, new Key("year"), new Value(2020))), + new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US"))))); + } + + @Test + public void tesNot() { + // isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"] + var exp = this.b.not(this.b.and(this.b.and(this.b.eq("isOpen", true), this.b.gte("year", 2020)), + this.b.in("country", "BG", "NL", "US"))) + .build(); + + assertThat(exp).isEqualTo(new Expression(NOT, + new Expression(AND, + new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)), + new Expression(GTE, new Key("year"), new Value(2020))), + new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))), + null)); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java new file mode 100644 index 000000000..218f29c97 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java @@ -0,0 +1,201 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.springframework.ai.vectorstore.filter.Filter.Expression; +import org.springframework.ai.vectorstore.filter.Filter.Group; +import org.springframework.ai.vectorstore.filter.Filter.Key; +import org.springframework.ai.vectorstore.filter.Filter.Value; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.AND; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.EQ; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.GTE; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.IN; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.LTE; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NE; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NIN; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NOT; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.OR; + +/** + * @author Christian Tzolov + */ +public class FilterExpressionTextParserTests { + + FilterExpressionTextParser parser = new FilterExpressionTextParser(); + + @Test + public void testEQ() { + // country == "BG" + Expression exp = this.parser.parse("country == 'BG'"); + assertThat(exp).isEqualTo(new Expression(EQ, new Key("country"), new Value("BG"))); + + assertThat(this.parser.getCache().get("WHERE " + "country == 'BG'")).isEqualTo(exp); + } + + @Test + public void tesEqAndGte() { + // genre == "drama" AND year >= 2020 + Expression exp = this.parser.parse("genre == 'drama' && year >= 2020"); + assertThat(exp).isEqualTo(new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")), + new Expression(GTE, new Key("year"), new Value(2020)))); + + assertThat(this.parser.getCache().get("WHERE " + "genre == 'drama' && year >= 2020")).isEqualTo(exp); + } + + @Test + public void tesIn() { + // genre in ["comedy", "documentary", "drama"] + Expression exp = this.parser.parse("genre in ['comedy', 'documentary', 'drama']"); + assertThat(exp) + .isEqualTo(new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama")))); + + assertThat(this.parser.getCache().get("WHERE " + "genre in ['comedy', 'documentary', 'drama']")).isEqualTo(exp); + } + + @Test + public void testNe() { + // year >= 2020 OR country == "BG" AND city != "Sofia" + Expression exp = this.parser.parse("year >= 2020 OR country == \"BG\" AND city != \"Sofia\""); + assertThat(exp).isEqualTo(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)), + new Expression(AND, new Expression(EQ, new Key("country"), new Value("BG")), + new Expression(NE, new Key("city"), new Value("Sofia"))))); + + assertThat(this.parser.getCache().get("WHERE " + "year >= 2020 OR country == \"BG\" AND city != \"Sofia\"")) + .isEqualTo(exp); + } + + @Test + public void testGroup() { + // (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"] + Expression exp = this.parser.parse("(year >= 2020 OR country == \"BG\") AND city NIN [\"Sofia\", \"Plovdiv\"]"); + + assertThat(exp).isEqualTo(new Expression(AND, + new Group(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)), + new Expression(EQ, new Key("country"), new Value("BG")))), + new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Plovdiv"))))); + + assertThat(this.parser.getCache() + .get("WHERE " + "(year >= 2020 OR country == \"BG\") AND city NIN [\"Sofia\", \"Plovdiv\"]")) + .isEqualTo(exp); + } + + @Test + public void tesBoolean() { + // isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"] + Expression exp = this.parser.parse("isOpen == true AND year >= 2020 AND country IN [\"BG\", \"NL\", \"US\"]"); + + assertThat(exp).isEqualTo(new Expression(AND, + new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)), + new Expression(GTE, new Key("year"), new Value(2020))), + new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US"))))); + assertThat(this.parser.getCache() + .get("WHERE " + "isOpen == true AND year >= 2020 AND country IN [\"BG\", \"NL\", \"US\"]")).isEqualTo(exp); + } + + @Test + public void tesNot() { + // NOT(isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]) + Expression exp = this.parser + .parse("not(isOpen == true AND year >= 2020 AND country IN [\"BG\", \"NL\", \"US\"])"); + + assertThat(exp).isEqualTo(new Expression(NOT, + new Group(new Expression(AND, + new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)), + new Expression(GTE, new Key("year"), new Value(2020))), + new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US"))))), + null)); + + assertThat(this.parser.getCache() + .get("WHERE " + "not(isOpen == true AND year >= 2020 AND country IN [\"BG\", \"NL\", \"US\"])")) + .isEqualTo(exp); + } + + @Test + public void tesNotNin() { + // NOT(country NOT IN ["BG", "NL", "US"]) + Expression exp = this.parser.parse("not(country NOT IN [\"BG\", \"NL\", \"US\"])"); + + assertThat(exp).isEqualTo(new Expression(NOT, + new Group(new Expression(NIN, new Key("country"), new Value(List.of("BG", "NL", "US")))), null)); + } + + @Test + public void tesNotNin2() { + // NOT country NOT IN ["BG", "NL", "US"] + Expression exp = this.parser.parse("NOT country NOT IN [\"BG\", \"NL\", \"US\"]"); + + assertThat(exp).isEqualTo(new Expression(NOT, + new Expression(NIN, new Key("country"), new Value(List.of("BG", "NL", "US"))), null)); + } + + @Test + public void tesNestedNot() { + // NOT(isOpen == true AND year >= 2020 AND NOT(country IN ["BG", "NL", "US"])) + Expression exp = this.parser + .parse("not(isOpen == true AND year >= 2020 AND NOT(country IN [\"BG\", \"NL\", \"US\"]))"); + + assertThat(exp).isEqualTo(new Expression(NOT, + new Group(new Expression(AND, + new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)), + new Expression(GTE, new Key("year"), new Value(2020))), + new Expression(NOT, + new Group(new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))), + null))), + null)); + + assertThat(this.parser.getCache() + .get("WHERE " + "not(isOpen == true AND year >= 2020 AND NOT(country IN [\"BG\", \"NL\", \"US\"]))")) + .isEqualTo(exp); + } + + @Test + public void testDecimal() { + // temperature >= -15.6 && temperature <= +20.13 + String expText = "temperature >= -15.6 && temperature <= +20.13"; + Expression exp = this.parser.parse(expText); + + assertThat(exp).isEqualTo(new Expression(AND, new Expression(GTE, new Key("temperature"), new Value(-15.6)), + new Expression(LTE, new Key("temperature"), new Value(20.13)))); + + assertThat(this.parser.getCache().get("WHERE " + expText)).isEqualTo(exp); + } + + @Test + public void testIdentifiers() { + Expression exp = this.parser.parse("'country.1' == 'BG'"); + assertThat(exp).isEqualTo(new Expression(EQ, new Key("'country.1'"), new Value("BG"))); + + exp = this.parser.parse("'country_1_2_3' == 'BG'"); + assertThat(exp).isEqualTo(new Expression(EQ, new Key("'country_1_2_3'"), new Value("BG"))); + + exp = this.parser.parse("\"country 1 2 3\" == 'BG'"); + assertThat(exp).isEqualTo(new Expression(EQ, new Key("\"country 1 2 3\""), new Value("BG"))); + } + + @Test + public void testUnescapedIdentifierWithUnderscores() { + Expression exp = this.parser.parse("file_name == 'medicaid-wa-faqs.pdf'"); + assertThat(exp).isEqualTo(new Expression(EQ, new Key("file_name"), new Value("medicaid-wa-faqs.pdf"))); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterHelperTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterHelperTests.java new file mode 100644 index 000000000..df793ecf0 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/FilterHelperTests.java @@ -0,0 +1,171 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.springframework.ai.vectorstore.filter.Filter.Expression; +import org.springframework.ai.vectorstore.filter.Filter.ExpressionType; +import org.springframework.ai.vectorstore.filter.Filter.Key; +import org.springframework.ai.vectorstore.filter.Filter.Value; +import org.springframework.ai.vectorstore.filter.converter.PrintFilterExpressionConverter; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +public class FilterHelperTests { + + @Test + public void negateEQ() { + assertThat(new FilterExpressionTextParser().parse("NOT key == 'UK' ")).isEqualTo(new Filter.Expression( + ExpressionType.NOT, new Filter.Expression(ExpressionType.EQ, new Key("key"), new Value("UK")), null)); + + assertThat(FilterHelper.negate(new FilterExpressionTextParser().parse("NOT key == 'UK' "))) + .isEqualTo(new Filter.Expression(ExpressionType.NE, new Key("key"), new Value("UK"))); + + assertThat(FilterHelper.negate(new FilterExpressionTextParser().parse("NOT (key == 'UK') "))) + .isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.NE, new Key("key"), new Value("UK")))); + } + + @Test + public void negateNE() { + var exp = new FilterExpressionTextParser().parse("NOT key != 'UK' "); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Expression(ExpressionType.EQ, new Key("key"), new Value("UK"))); + + } + + @Test + public void negateGT() { + var exp = new FilterExpressionTextParser().parse("NOT key > 13 "); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Expression(ExpressionType.LTE, new Key("key"), new Value(13))); + + } + + @Test + public void negateGTE() { + var exp = new FilterExpressionTextParser().parse("NOT key >= 13 "); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(13))); + } + + @Test + public void negateLT() { + var exp = new FilterExpressionTextParser().parse("NOT key < 13 "); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Expression(ExpressionType.GTE, new Key("key"), new Value(13))); + } + + @Test + public void negateLTE() { + var exp = new FilterExpressionTextParser().parse("NOT key <= 13 "); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Expression(ExpressionType.GT, new Key("key"), new Value(13))); + } + + @Test + public void negateIN() { + var exp = new FilterExpressionTextParser().parse("NOT key IN [11, 12, 13] "); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Expression(ExpressionType.NIN, new Key("key"), new Value(List.of(11, 12, 13)))); + } + + @Test + public void negateNIN() { + var exp = new FilterExpressionTextParser().parse("NOT key NIN [11, 12, 13] "); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Expression(ExpressionType.IN, new Key("key"), new Value(List.of(11, 12, 13)))); + } + + @Test + public void negateNIN2() { + var exp = new FilterExpressionTextParser().parse("NOT key NOT IN [11, 12, 13] "); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Expression(ExpressionType.IN, new Key("key"), new Value(List.of(11, 12, 13)))); + } + + @Test + public void negateAND() { + var exp = new FilterExpressionTextParser().parse("NOT(key >= 11 AND key < 13)"); + assertThat(FilterHelper.negate(exp)).isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.OR, + new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(11)), + new Filter.Expression(ExpressionType.GTE, new Key("key"), new Value(13))))); + } + + @Test + public void negateOR() { + var exp = new FilterExpressionTextParser().parse("NOT(key >= 11 OR key < 13)"); + assertThat(FilterHelper.negate(exp)).isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.AND, + new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(11)), + new Filter.Expression(ExpressionType.GTE, new Key("key"), new Value(13))))); + } + + @Test + public void negateNot() { + var exp = new FilterExpressionTextParser().parse("NOT NOT(key >= 11)"); + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(11)))); + } + + @Test + public void negateNestedNot() { + var exp = new FilterExpressionTextParser().parse("NOT(NOT(key >= 11))"); + assertThat(exp).isEqualTo( + new Filter.Expression(ExpressionType.NOT, new Filter.Group(new Filter.Expression(ExpressionType.NOT, + new Filter.Group(new Filter.Expression(ExpressionType.GTE, new Key("key"), new Value(11))))))); + + assertThat(FilterHelper.negate(exp)) + .isEqualTo(new Filter.Group(new Filter.Expression(ExpressionType.LT, new Key("key"), new Value(11)))); + } + + @Test + public void expandIN() { + var exp = new FilterExpressionTextParser().parse("key IN [11, 12, 13] "); + assertThat(new InNinTestConverter().convertExpression(exp)).isEqualTo("key EQ 11 OR key EQ 12 OR key EQ 13"); + } + + @Test + public void expandNIN() { + var exp1 = new FilterExpressionTextParser().parse("key NIN [11, 12, 13] "); + var exp2 = new FilterExpressionTextParser().parse("key NOT IN [11, 12, 13] "); + assertThat(exp1).isEqualTo(exp2); + assertThat(new InNinTestConverter().convertExpression(exp1)).isEqualTo("key NE 11 AND key NE 12 AND key NE 13"); + } + + private static class InNinTestConverter extends PrintFilterExpressionConverter { + + @Override + public void doExpression(Expression expression, StringBuilder context) { + if (expression.type() == ExpressionType.IN) { + FilterHelper.expandIn(expression, context, this); + } + else if (expression.type() == ExpressionType.NIN) { + FilterHelper.expandNin(expression, context, this); + } + else { + super.doExpression(expression, context); + } + } + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/SearchRequestTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/SearchRequestTests.java new file mode 100644 index 000000000..acdd0b3ed --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/SearchRequestTests.java @@ -0,0 +1,139 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +import org.junit.jupiter.api.Test; + +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.filter.FilterExpressionTextParser.FilterExpressionParseException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * @author Christian Tzolov + */ +public class SearchRequestTests { + + @Test + public void createDefaults() { + var emptyRequest = SearchRequest.defaults(); + assertThat(emptyRequest.getQuery()).isEqualTo(""); + checkDefaults(emptyRequest); + } + + @Test + public void createQuery() { + var emptyRequest = SearchRequest.query("New Query"); + assertThat(emptyRequest.getQuery()).isEqualTo("New Query"); + checkDefaults(emptyRequest); + } + + @Test + public void createFrom() { + var originalRequest = SearchRequest.query("New Query") + .withTopK(696) + .withSimilarityThreshold(0.678) + .withFilterExpression("country == 'NL'"); + + var newRequest = SearchRequest.from(originalRequest); + + assertThat(newRequest).isNotSameAs(originalRequest); + assertThat(newRequest.getQuery()).isEqualTo(originalRequest.getQuery()); + assertThat(newRequest.getTopK()).isEqualTo(originalRequest.getTopK()); + assertThat(newRequest.getFilterExpression()).isEqualTo(originalRequest.getFilterExpression()); + assertThat(newRequest.getSimilarityThreshold()).isEqualTo(originalRequest.getSimilarityThreshold()); + } + + @Test + public void withQuery() { + var emptyRequest = SearchRequest.defaults(); + assertThat(emptyRequest.getQuery()).isEqualTo(""); + + emptyRequest.withQuery("New Query"); + assertThat(emptyRequest.getQuery()).isEqualTo("New Query"); + } + + @Test + public void withSimilarityThreshold() { + var request = SearchRequest.query("Test").withSimilarityThreshold(0.678); + assertThat(request.getSimilarityThreshold()).isEqualTo(0.678); + + request.withSimilarityThreshold(0.9); + assertThat(request.getSimilarityThreshold()).isEqualTo(0.9); + + assertThatThrownBy(() -> request.withSimilarityThreshold(-1)).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Similarity threshold must be in [0,1] range."); + + assertThatThrownBy(() -> request.withSimilarityThreshold(1.1)).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Similarity threshold must be in [0,1] range."); + + } + + @Test + public void withTopK() { + var request = SearchRequest.query("Test").withTopK(66); + assertThat(request.getTopK()).isEqualTo(66); + + request.withTopK(89); + assertThat(request.getTopK()).isEqualTo(89); + + assertThatThrownBy(() -> request.withTopK(-1)).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("TopK should be positive."); + + } + + @Test + public void withFilterExpression() { + + var request = SearchRequest.query("Test").withFilterExpression("country == 'BG' && year >= 2022"); + assertThat(request.getFilterExpression()).isEqualTo(new Filter.Expression(Filter.ExpressionType.AND, + new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("country"), new Filter.Value("BG")), + new Filter.Expression(Filter.ExpressionType.GTE, new Filter.Key("year"), new Filter.Value(2022)))); + assertThat(request.hasFilterExpression()).isTrue(); + + request.withFilterExpression("active == true"); + assertThat(request.getFilterExpression()).isEqualTo( + new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("active"), new Filter.Value(true))); + assertThat(request.hasFilterExpression()).isTrue(); + + request.withFilterExpression(new FilterExpressionBuilder().eq("country", "NL").build()); + assertThat(request.getFilterExpression()).isEqualTo( + new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("country"), new Filter.Value("NL"))); + assertThat(request.hasFilterExpression()).isTrue(); + + request.withFilterExpression((String) null); + assertThat(request.getFilterExpression()).isNull(); + assertThat(request.hasFilterExpression()).isFalse(); + + request.withFilterExpression((Filter.Expression) null); + assertThat(request.getFilterExpression()).isNull(); + assertThat(request.hasFilterExpression()).isFalse(); + + assertThatThrownBy(() -> request.withFilterExpression("FooBar")) + .isInstanceOf(FilterExpressionParseException.class) + .hasMessageContaining("Error: no viable alternative at input 'FooBar'"); + + } + + private void checkDefaults(SearchRequest request) { + assertThat(request.getFilterExpression()).isNull(); + assertThat(request.getSimilarityThreshold()).isEqualTo(SearchRequest.SIMILARITY_THRESHOLD_ACCEPT_ALL); + assertThat(request.getTopK()).isEqualTo(SearchRequest.DEFAULT_TOP_K); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverterTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverterTests.java new file mode 100644 index 000000000..9fc858aa1 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverterTests.java @@ -0,0 +1,126 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.converter; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.springframework.ai.vectorstore.filter.Filter.Expression; +import org.springframework.ai.vectorstore.filter.Filter.Group; +import org.springframework.ai.vectorstore.filter.Filter.Key; +import org.springframework.ai.vectorstore.filter.Filter.Value; +import org.springframework.ai.vectorstore.filter.FilterExpressionConverter; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.AND; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.EQ; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.GTE; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.IN; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.LTE; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NE; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NIN; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.OR; + +/** + * @author Christian Tzolov + */ +public class PineconeFilterExpressionConverterTests { + + FilterExpressionConverter converter = new PineconeFilterExpressionConverter(); + + @Test + public void testEQ() { + // country == "BG" + String vectorExpr = this.converter.convertExpression(new Expression(EQ, new Key("country"), new Value("BG"))); + assertThat(vectorExpr).isEqualTo("{\"country\": {\"$eq\": \"BG\"}}"); + } + + @Test + public void tesEqAndGte() { + // genre == "drama" AND year >= 2020 + String vectorExpr = this.converter + .convertExpression(new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")), + new Expression(GTE, new Key("year"), new Value(2020)))); + assertThat(vectorExpr) + .isEqualTo("{\"$and\": [{\"genre\": {\"$eq\": \"drama\"}},{\"year\": {\"$gte\": 2020}}]}"); + } + + @Test + public void tesIn() { + // genre in ["comedy", "documentary", "drama"] + String vectorExpr = this.converter.convertExpression( + new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama")))); + assertThat(vectorExpr).isEqualTo("{\"genre\": {\"$in\": [\"comedy\",\"documentary\",\"drama\"]}}"); + } + + @Test + public void testNe() { + // year >= 2020 OR country == "BG" AND city != "Sofia" + String vectorExpr = this.converter + .convertExpression(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)), + new Expression(AND, new Expression(EQ, new Key("country"), new Value("BG")), + new Expression(NE, new Key("city"), new Value("Sofia"))))); + assertThat(vectorExpr).isEqualTo( + "{\"$or\": [{\"year\": {\"$gte\": 2020}},{\"$and\": [{\"country\": {\"$eq\": \"BG\"}},{\"city\": {\"$ne\": \"Sofia\"}}]}]}"); + } + + @Test + public void testGroup() { + // (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"] + String vectorExpr = this.converter.convertExpression(new Expression(AND, + new Group(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)), + new Expression(EQ, new Key("country"), new Value("BG")))), + new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Plovdiv"))))); + assertThat(vectorExpr).isEqualTo( + "{\"$and\": [{\"$or\": [{\"year\": {\"$gte\": 2020}},{\"country\": {\"$eq\": \"BG\"}}]},{\"city\": {\"$nin\": [\"Sofia\",\"Plovdiv\"]}}]}"); + } + + @Test + public void tesBoolean() { + // isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"] + String vectorExpr = this.converter.convertExpression(new Expression(AND, + new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)), + new Expression(GTE, new Key("year"), new Value(2020))), + new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US"))))); + + assertThat(vectorExpr).isEqualTo( + "{\"$and\": [{\"$and\": [{\"isOpen\": {\"$eq\": true}},{\"year\": {\"$gte\": 2020}}]},{\"country\": {\"$in\": [\"BG\",\"NL\",\"US\"]}}]}"); + } + + @Test + public void testDecimal() { + // temperature >= -15.6 && temperature <= +20.13 + String vectorExpr = this.converter + .convertExpression(new Expression(AND, new Expression(GTE, new Key("temperature"), new Value(-15.6)), + new Expression(LTE, new Key("temperature"), new Value(20.13)))); + + assertThat(vectorExpr) + .isEqualTo("{\"$and\": [{\"temperature\": {\"$gte\": -15.6}},{\"temperature\": {\"$lte\": 20.13}}]}"); + } + + @Test + public void testComplexIdentifiers() { + String vectorExpr = this.converter + .convertExpression(new Expression(EQ, new Key("\"country 1 2 3\""), new Value("BG"))); + assertThat(vectorExpr).isEqualTo("{\"country 1 2 3\": {\"$eq\": \"BG\"}}"); + + vectorExpr = this.converter.convertExpression(new Expression(EQ, new Key("'country 1 2 3'"), new Value("BG"))); + assertThat(vectorExpr).isEqualTo("{\"country 1 2 3\": {\"$eq\": \"BG\"}}"); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConventionTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConventionTests.java new file mode 100644 index 000000000..981ac04b1 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConventionTests.java @@ -0,0 +1,130 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import java.util.List; + +import io.micrometer.common.KeyValue; +import io.micrometer.observation.Observation; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.document.Document; +import org.springframework.ai.observation.conventions.SpringAiKind; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.HighCardinalityKeyNames; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.LowCardinalityKeyNames; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link DefaultVectorStoreObservationConvention}. + * + * @author Christian Tzolov + * @author Thomas Vitale + */ +class DefaultVectorStoreObservationConventionTests { + + private final DefaultVectorStoreObservationConvention observationConvention = new DefaultVectorStoreObservationConvention(); + + @Test + void shouldHaveName() { + assertThat(this.observationConvention.getName()) + .isEqualTo(DefaultVectorStoreObservationConvention.DEFAULT_NAME); + } + + @Test + void shouldHaveContextualName() { + VectorStoreObservationContext observationContext = VectorStoreObservationContext + .builder("my-database", VectorStoreObservationContext.Operation.QUERY) + .build(); + assertThat(this.observationConvention.getContextualName(observationContext)).isEqualTo("my-database query"); + } + + @Test + void supportsOnlyVectorStoreObservationContext() { + VectorStoreObservationContext observationContext = VectorStoreObservationContext + .builder("my-database", VectorStoreObservationContext.Operation.QUERY) + .build(); + assertThat(this.observationConvention.supportsContext(observationContext)).isTrue(); + assertThat(this.observationConvention.supportsContext(new Observation.Context())).isFalse(); + } + + @Test + void shouldHaveRequiredKeyValues() { + VectorStoreObservationContext observationContext = VectorStoreObservationContext + .builder("my_database", VectorStoreObservationContext.Operation.QUERY) + .build(); + assertThat(this.observationConvention.getLowCardinalityKeyValues(observationContext)).contains( + KeyValue.of(LowCardinalityKeyNames.SPRING_AI_KIND.asString(), SpringAiKind.VECTOR_STORE.value()), + KeyValue.of(LowCardinalityKeyNames.DB_OPERATION_NAME.asString(), "query"), + KeyValue.of(LowCardinalityKeyNames.DB_SYSTEM.asString(), "my_database")); + } + + @Test + void shouldHaveOptionalKeyValues() { + VectorStoreObservationContext observationContext = VectorStoreObservationContext + .builder("my-database", VectorStoreObservationContext.Operation.QUERY) + .withCollectionName("COLLECTION_NAME") + .withDimensions(696) + .withFieldName("FIELD_NAME") + .withNamespace("NAMESPACE") + .withSimilarityMetric("SIMILARITY_METRIC") + .withQueryRequest(SearchRequest.query("VDB QUERY").withFilterExpression("country == 'UK' && year >= 2020")) + .build(); + + List queryResponseDocs = List.of(new Document("doc1"), new Document("doc2")); + + observationContext.setQueryResponse(queryResponseDocs); + + assertThat(this.observationConvention.getLowCardinalityKeyValues(observationContext)) + .contains(KeyValue.of(LowCardinalityKeyNames.DB_OPERATION_NAME.asString(), + VectorStoreObservationContext.Operation.QUERY.value)); + + // Optional, filter only added content + assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)) + .doesNotContain(KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_QUERY_RESPONSE_DOCUMENTS, "[doc1,doc2]")); + + assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)).contains( + KeyValue.of(HighCardinalityKeyNames.DB_COLLECTION_NAME.asString(), "COLLECTION_NAME"), + KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_DIMENSION_COUNT.asString(), "696"), + KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_FIELD_NAME.asString(), "FIELD_NAME"), + KeyValue.of(HighCardinalityKeyNames.DB_NAMESPACE.asString(), "NAMESPACE"), + KeyValue.of(HighCardinalityKeyNames.DB_SEARCH_SIMILARITY_METRIC.asString(), "SIMILARITY_METRIC"), + KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_QUERY_CONTENT.asString(), "VDB QUERY"), + KeyValue.of(HighCardinalityKeyNames.DB_VECTOR_QUERY_FILTER.asString(), + "Expression[type=AND, left=Expression[type=EQ, left=Key[key=country], right=Value[value=UK]], right=Expression[type=GTE, left=Key[key=year], right=Value[value=2020]]]")); + } + + @Test + void shouldNotHaveKeyValuesWhenMissing() { + VectorStoreObservationContext observationContext = VectorStoreObservationContext + .builder("my-database", VectorStoreObservationContext.Operation.QUERY) + .build(); + + assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext) + .stream() + .map(KeyValue::getKey) + .toList()).doesNotContain(HighCardinalityKeyNames.DB_COLLECTION_NAME.asString(), + HighCardinalityKeyNames.DB_VECTOR_DIMENSION_COUNT.asString(), + HighCardinalityKeyNames.DB_VECTOR_FIELD_NAME.asString(), + HighCardinalityKeyNames.DB_NAMESPACE.asString(), + HighCardinalityKeyNames.DB_SEARCH_SIMILARITY_METRIC.asString(), + HighCardinalityKeyNames.DB_VECTOR_QUERY_CONTENT.asString(), + HighCardinalityKeyNames.DB_VECTOR_QUERY_FILTER.asString()); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContextTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContextTests.java new file mode 100644 index 000000000..6f6abd873 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContextTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Unit tests for {@link VectorStoreObservationContext}. + * + * @author Christian Tzolov + */ +class VectorStoreObservationContextTests { + + @Test + void whenMandatoryFieldsThenReturn() { + var observationContext = VectorStoreObservationContext + .builder("db", VectorStoreObservationContext.Operation.ADD) + .build(); + assertThat(observationContext).isNotNull(); + } + + @Test + void whenDbSystemIsNullThenThrow() { + assertThatThrownBy(() -> VectorStoreObservationContext.builder(null, "delete").build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("databaseSystem cannot be null or empty"); + } + + @Test + void whenOperationNameIsNullThenThrow() { + assertThatThrownBy(() -> VectorStoreObservationContext.builder("Db", "").build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("operationName cannot be null or empty"); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilterTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilterTests.java new file mode 100644 index 000000000..ba7a37e05 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilterTests.java @@ -0,0 +1,73 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import java.util.List; + +import io.micrometer.common.KeyValue; +import io.micrometer.observation.Observation; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.HighCardinalityKeyNames; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link VectorStoreQueryResponseObservationFilter}. + * + * @author Christian Tzolov + * @author Thomas Vitale + */ +class VectorStoreQueryResponseObservationFilterTests { + + private final VectorStoreQueryResponseObservationFilter observationFilter = new VectorStoreQueryResponseObservationFilter(); + + @Test + void whenNotSupportedObservationContextThenReturnOriginalContext() { + var expectedContext = new Observation.Context(); + var actualContext = this.observationFilter.map(expectedContext); + + assertThat(actualContext).isEqualTo(expectedContext); + } + + @Test + void whenEmptyQueryResponseThenReturnOriginalContext() { + var expectedContext = VectorStoreObservationContext.builder("db", VectorStoreObservationContext.Operation.ADD) + .build(); + + var actualContext = this.observationFilter.map(expectedContext); + + assertThat(actualContext).isEqualTo(expectedContext); + } + + @Test + void whenNonEmptyQueryResponseThenAugmentContext() { + var expectedContext = VectorStoreObservationContext.builder("db", VectorStoreObservationContext.Operation.ADD) + .build(); + + List queryResponseDocs = List.of(new Document("doc1"), new Document("doc2")); + + expectedContext.setQueryResponse(queryResponseDocs); + + var augmentedContext = this.observationFilter.map(expectedContext); + + assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue + .of(HighCardinalityKeyNames.DB_VECTOR_QUERY_RESPONSE_DOCUMENTS.asString(), "[\"doc1\", \"doc2\"]")); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandlerTests.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandlerTests.java new file mode 100644 index 000000000..499c3cc02 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/src/test/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandlerTests.java @@ -0,0 +1,71 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import java.util.List; + +import io.micrometer.tracing.handler.TracingObservationHandler; +import io.micrometer.tracing.otel.bridge.OtelCurrentTraceContext; +import io.micrometer.tracing.otel.bridge.OtelTracer; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.document.Document; +import org.springframework.ai.observation.conventions.VectorStoreObservationAttributes; +import org.springframework.ai.observation.conventions.VectorStoreObservationEventNames; +import org.springframework.ai.observation.tracing.TracingHelper; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link VectorStoreQueryResponseObservationHandler}. + * + * @author Thomas Vitale + */ +class VectorStoreQueryResponseObservationHandlerTests { + + @Test + void whenCompletionWithTextThenSpanEvent() { + var observationContext = VectorStoreObservationContext + .builder("db", VectorStoreObservationContext.Operation.ADD) + .withQueryResponse(List.of(new Document("hello"), new Document("other-side"))) + .build(); + var sdkTracer = SdkTracerProvider.builder().build().get("test"); + var otelTracer = new OtelTracer(sdkTracer, new OtelCurrentTraceContext(), null); + var span = otelTracer.nextSpan(); + var tracingContext = new TracingObservationHandler.TracingContext(); + tracingContext.setSpan(span); + observationContext.put(TracingObservationHandler.TracingContext.class, tracingContext); + + new VectorStoreQueryResponseObservationHandler().onStop(observationContext); + + var otelSpan = TracingHelper.extractOtelSpan(tracingContext); + assertThat(otelSpan).isNotNull(); + var spanData = ((ReadableSpan) otelSpan).toSpanData(); + assertThat(spanData.getEvents().size()).isEqualTo(1); + assertThat(spanData.getEvents().get(0).getName()) + .isEqualTo(VectorStoreObservationEventNames.CONTENT_QUERY_RESPONSE.value()); + assertThat(spanData.getEvents() + .get(0) + .getAttributes() + .get(AttributeKey.stringArrayKey(VectorStoreObservationAttributes.DB_VECTOR_QUERY_CONTENT.value()))) + .containsOnly("hello", "other-side"); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/AbstractVectorStoreBuilder.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/AbstractVectorStoreBuilder.java new file mode 100644 index 000000000..0affbdcc8 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/AbstractVectorStoreBuilder.java @@ -0,0 +1,90 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore; + +import io.micrometer.observation.ObservationRegistry; + +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Abstract base builder implementing common builder functionality for + * {@link VectorStore}. Provides default implementations for observation-related settings. + * + * @param the concrete builder type, enabling method chaining with the correct return + * type + */ +public abstract class AbstractVectorStoreBuilder> + implements VectorStore.Builder { + + protected EmbeddingModel embeddingModel; + + protected ObservationRegistry observationRegistry = ObservationRegistry.NOOP; + + @Nullable + protected VectorStoreObservationConvention customObservationConvention; + + public EmbeddingModel getEmbeddingModel() { + return this.embeddingModel; + } + + public ObservationRegistry getObservationRegistry() { + return this.observationRegistry; + } + + @Nullable + public VectorStoreObservationConvention getCustomObservationConvention() { + return this.customObservationConvention; + } + + /** + * Returns this builder cast to the concrete builder type. Used internally to enable + * proper method chaining in subclasses. + * @return this builder cast to the concrete type + */ + @SuppressWarnings("unchecked") + protected T self() { + return (T) this; + } + + @Override + public T observationRegistry(ObservationRegistry observationRegistry) { + Assert.notNull(observationRegistry, "ObservationRegistry must not be null"); + this.observationRegistry = observationRegistry; + return self(); + } + + @Override + public T customObservationConvention(VectorStoreObservationConvention convention) { + this.customObservationConvention = convention; + return self(); + } + + @Override + public T embeddingModel(EmbeddingModel embeddingModel) { + Assert.notNull(embeddingModel, "EmbeddingModel must not be null"); + this.embeddingModel = embeddingModel; + return self(); + } + + protected void validate() { + Assert.notNull(this.embeddingModel, "EmbeddingModel must be configured"); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SearchRequest.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SearchRequest.java new file mode 100644 index 000000000..597b39df0 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SearchRequest.java @@ -0,0 +1,283 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore; + +import java.util.Objects; + +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.filter.Filter; +import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder; +import org.springframework.ai.vectorstore.filter.FilterExpressionTextParser; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Similarity search request builder. Use the {@link #query(String)}, {@link #defaults()} + * or {@link #from(SearchRequest)} factory methods to create a new {@link SearchRequest} + * instance and then apply the 'with' methods to alter the default values. + * + * @author Christian Tzolov + * @author Thomas Vitale + */ +public final class SearchRequest { + + /** + * Similarity threshold that accepts all search scores. A threshold value of 0.0 means + * any similarity is accepted or disable the similarity threshold filtering. A + * threshold value of 1.0 means an exact match is required. + */ + public static final double SIMILARITY_THRESHOLD_ACCEPT_ALL = 0.0; + + /** + * Default value for the top 'k' similar results to return. + */ + public static final int DEFAULT_TOP_K = 4; + + private String query; + + private int topK = DEFAULT_TOP_K; + + private double similarityThreshold = SIMILARITY_THRESHOLD_ACCEPT_ALL; + + @Nullable + private Filter.Expression filterExpression; + + private SearchRequest(String query) { + this.query = query; + } + + /** + * Create a new {@link SearchRequest} builder instance with specified embedding query + * string. + * @param query Text to use for embedding similarity comparison. + * @return Returns new {@link SearchRequest} builder instance. + */ + public static SearchRequest query(String query) { + Assert.notNull(query, "Query can not be null."); + return new SearchRequest(query); + } + + /** + * Create a new {@link SearchRequest} builder instance with an empty embedding query + * string. Use the {@link #withQuery(String query)} to set/update the embedding query + * text. + * @return Returns new {@link SearchRequest} builder instance. + */ + public static SearchRequest defaults() { + return new SearchRequest(""); + } + + /** + * Copy an existing {@link SearchRequest} instance. + * @param originalSearchRequest {@link SearchRequest} instance to copy. + * @return Returns new {@link SearchRequest} builder instance. + */ + public static SearchRequest from(SearchRequest originalSearchRequest) { + return new SearchRequest(originalSearchRequest.getQuery()).withTopK(originalSearchRequest.getTopK()) + .withSimilarityThreshold(originalSearchRequest.getSimilarityThreshold()) + .withFilterExpression(originalSearchRequest.getFilterExpression()); + } + + /** + * @param query Text to use for embedding similarity comparison. + * @return this builder. + */ + public SearchRequest withQuery(String query) { + Assert.notNull(query, "Query can not be null."); + this.query = query; + return this; + } + + /** + * @param topK the top 'k' similar results to return. + * @return this builder. + */ + public SearchRequest withTopK(int topK) { + Assert.isTrue(topK >= 0, "TopK should be positive."); + this.topK = topK; + return this; + } + + /** + * Similarity threshold score to filter the search response by. Only documents with + * similarity score equal or greater than the 'threshold' will be returned. Note that + * this is a post-processing step performed on the client not the server side. A + * threshold value of 0.0 means any similarity is accepted or disable the similarity + * threshold filtering. A threshold value of 1.0 means an exact match is required. + * @param threshold The lower bound of the similarity score. + * @return this builder. + */ + public SearchRequest withSimilarityThreshold(double threshold) { + Assert.isTrue(threshold >= 0 && threshold <= 1, "Similarity threshold must be in [0,1] range."); + this.similarityThreshold = threshold; + return this; + } + + /** + * Sets disables the similarity threshold by setting it to 0.0 - all results are + * accepted. + * @return this builder. + */ + public SearchRequest withSimilarityThresholdAll() { + return withSimilarityThreshold(SIMILARITY_THRESHOLD_ACCEPT_ALL); + } + + /** + * Retrieves documents by query embedding similarity and matching the filters. Value + * of 'null' means that no metadata filters will be applied to the search. + * + * For example if the {@link Document#getMetadata()} schema is: + * + * 
{@code
+	 * {
+	 * "country": ,
+	 * "city": ,
+	 * "year": ,
+	 * "price": ,
+	 * "isActive": 
+	 * }
+	 * }
+ * + * you can constrain the search result to only UK countries with isActive=true and + * year equal or greater 2020. You can build this such metadata filter + * programmatically like this: + * + * 
{@code
+	 * var exp = new Filter.Expression(AND,
+	 * 		new Expression(EQ, new Key("country"), new Value("UK")),
+	 * 		new Expression(AND,
+	 * 				new Expression(GTE, new Key("year"), new Value(2020)),
+	 * 				new Expression(EQ, new Key("isActive"), new Value(true))));
+	 * }
+ * + * The {@link Filter.Expression} is portable across all vector stores.
+ * + * + * The {@link FilterExpressionBuilder} is a DSL creating expressions programmatically: + * + * 
{@code
+	 * var b = new FilterExpressionBuilder();
+	 * var exp = b.and(
+	 * 		b.eq("country", "UK"),
+	 * 		b.and(
+	 * 			b.gte("year", 2020),
+	 * 			b.eq("isActive", true)));
+	 * }
+ * + * The {@link FilterExpressionTextParser} converts textual, SQL like filter expression + * language into {@link Filter.Expression}: + * + * 
{@code
+	 * var parser = new FilterExpressionTextParser();
+	 * var exp = parser.parse("country == 'UK' && isActive == true && year >=2020");
+	 * }
+ * @param expression {@link Filter.Expression} instance used to define the metadata + * filter criteria. The 'null' value stands for no expression filters. + * @return this builder. + */ + public SearchRequest withFilterExpression(@Nullable Filter.Expression expression) { + this.filterExpression = expression; + return this; + } + + /** + * Document metadata filter expression. For example if your + * {@link Document#getMetadata()} has a schema like: + * + * 
{@code
+	 * {
+	 * "country": ,
+	 * "city": ,
+	 * "year": ,
+	 * "price": ,
+	 * "isActive": 
+	 * }
+	 * }
+ * + * then you can constrain the search result with metadata filter expressions like: + * + * 
{@code
+	 * country == 'UK' && year >= 2020 && isActive == true
+	 * Or
+	 * country == 'BG' && (city NOT IN ['Sofia', 'Plovdiv'] || price < 134.34)
+	 * }
+ * + * This ensures that the response contains only embeddings that match the specified + * filer criteria.
+ * + * The declarative, SQL like, filter syntax is portable across all vector stores + * supporting the filter search feature.
+ * + * The {@link FilterExpressionTextParser} is used to convert the text filter + * expression into {@link Filter.Expression}. + * @param textExpression declarative, portable, SQL like, metadata filter syntax. The + * 'null' value stands for no expression filters. + * @return this.builder + */ + public SearchRequest withFilterExpression(@Nullable String textExpression) { + this.filterExpression = (textExpression != null) ? new FilterExpressionTextParser().parse(textExpression) + : null; + return this; + } + + public String getQuery() { + return this.query; + } + + public int getTopK() { + return this.topK; + } + + public double getSimilarityThreshold() { + return this.similarityThreshold; + } + + @Nullable + public Filter.Expression getFilterExpression() { + return this.filterExpression; + } + + public boolean hasFilterExpression() { + return this.filterExpression != null; + } + + @Override + public String toString() { + return "SearchRequest{" + "query='" + this.query + '\'' + ", topK=" + this.topK + ", similarityThreshold=" + + this.similarityThreshold + ", filterExpression=" + this.filterExpression + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SearchRequest that = (SearchRequest) o; + return this.topK == that.topK && Double.compare(that.similarityThreshold, this.similarityThreshold) == 0 + && Objects.equals(this.query, that.query) + && Objects.equals(this.filterExpression, that.filterExpression); + } + + @Override + public int hashCode() { + return Objects.hash(this.query, this.topK, this.similarityThreshold, this.filterExpression); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java new file mode 100644 index 000000000..28cbfc2eb --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java @@ -0,0 +1,283 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.json.JsonMapper; +import io.micrometer.observation.ObservationRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.ai.document.Document; +import org.springframework.ai.document.DocumentMetadata; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.observation.conventions.VectorStoreProvider; +import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric; +import org.springframework.ai.util.JacksonUtils; +import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention; +import org.springframework.core.io.Resource; + +/** + * SimpleVectorStore is a simple implementation of the VectorStore interface. + * + * It also provides methods to save the current state of the vectors to a file, and to + * load vectors from a file. + * + * For a deeper understanding of the mathematical concepts and computations involved in + * calculating similarity scores among vectors, refer to this + * [resource](https://docs.spring.io/spring-ai/reference/api/vectordbs.html#_understanding_vectors). + * + * @author Raphael Yu + * @author Dingmeng Xue + * @author Mark Pollack + * @author Christian Tzolov + * @author Sebastien Deleuze + * @author Ilayaperumal Gopinathan + * @author Thomas Vitale + */ +public class SimpleVectorStore extends AbstractObservationVectorStore { + + private static final Logger logger = LoggerFactory.getLogger(SimpleVectorStore.class); + + private final ObjectMapper objectMapper; + + protected Map store = new ConcurrentHashMap<>(); + + protected EmbeddingModel embeddingModel; + + public SimpleVectorStore(EmbeddingModel embeddingModel) { + this(embeddingModel, ObservationRegistry.NOOP, null); + } + + public SimpleVectorStore(EmbeddingModel embeddingModel, ObservationRegistry observationRegistry, + VectorStoreObservationConvention customObservationConvention) { + + super(observationRegistry, customObservationConvention); + + Objects.requireNonNull(embeddingModel, "EmbeddingModel must not be null"); + this.embeddingModel = embeddingModel; + this.objectMapper = JsonMapper.builder().addModules(JacksonUtils.instantiateAvailableModules()).build(); + } + + @Override + public void doAdd(List documents) { + Objects.requireNonNull(documents, "Documents list cannot be null"); + if (documents.isEmpty()) { + throw new IllegalArgumentException("Documents list cannot be empty"); + } + + for (Document document : documents) { + logger.info("Calling EmbeddingModel for document id = {}", document.getId()); + float[] embedding = this.embeddingModel.embed(document); + SimpleVectorStoreContent storeContent = new SimpleVectorStoreContent(document.getId(), + document.getContent(), document.getMetadata(), embedding); + this.store.put(document.getId(), storeContent); + } + } + + @Override + public Optional doDelete(List idList) { + for (String id : idList) { + this.store.remove(id); + } + return Optional.of(true); + } + + @Override + public List doSimilaritySearch(SearchRequest request) { + if (request.getFilterExpression() != null) { + throw new UnsupportedOperationException( + "The [" + this.getClass() + "] doesn't support metadata filtering!"); + } + + float[] userQueryEmbedding = getUserQueryEmbedding(request.getQuery()); + return this.store.values() + .stream() + .map(content -> content + .toDocument(EmbeddingMath.cosineSimilarity(userQueryEmbedding, content.getEmbedding()))) + .filter(document -> document.getScore() >= request.getSimilarityThreshold()) + .sorted(Comparator.comparing(Document::getScore).reversed()) + .limit(request.getTopK()) + .toList(); + } + + /** + * Serialize the vector store content into a file in JSON format. + * @param file the file to save the vector store content + */ + public void save(File file) { + String json = getVectorDbAsJson(); + try { + if (!file.exists()) { + logger.info("Creating new vector store file: {}", file); + try { + Files.createFile(file.toPath()); + } + catch (FileAlreadyExistsException e) { + throw new RuntimeException("File already exists: " + file, e); + } + catch (IOException e) { + throw new RuntimeException("Failed to create new file: " + file + ". Reason: " + e.getMessage(), e); + } + } + else { + logger.info("Overwriting existing vector store file: {}", file); + } + try (OutputStream stream = new FileOutputStream(file); + Writer writer = new OutputStreamWriter(stream, StandardCharsets.UTF_8)) { + writer.write(json); + writer.flush(); + } + } + catch (IOException ex) { + logger.error("IOException occurred while saving vector store file.", ex); + throw new RuntimeException(ex); + } + catch (SecurityException ex) { + logger.error("SecurityException occurred while saving vector store file.", ex); + throw new RuntimeException(ex); + } + catch (NullPointerException ex) { + logger.error("NullPointerException occurred while saving vector store file.", ex); + throw new RuntimeException(ex); + } + } + + /** + * Deserialize the vector store content from a file in JSON format into memory. + * @param file the file to load the vector store content + */ + public void load(File file) { + TypeReference> typeRef = new TypeReference<>() { + + }; + try { + this.store = this.objectMapper.readValue(file, typeRef); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + /** + * Deserialize the vector store content from a resource in JSON format into memory. + * @param resource the resource to load the vector store content + */ + public void load(Resource resource) { + TypeReference> typeRef = new TypeReference<>() { + + }; + try { + this.store = this.objectMapper.readValue(resource.getInputStream(), typeRef); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + private String getVectorDbAsJson() { + ObjectWriter objectWriter = this.objectMapper.writerWithDefaultPrettyPrinter(); + String json; + try { + json = objectWriter.writeValueAsString(this.store); + } + catch (JsonProcessingException e) { + throw new RuntimeException("Error serializing documentMap to JSON.", e); + } + return json; + } + + private float[] getUserQueryEmbedding(String query) { + return this.embeddingModel.embed(query); + } + + @Override + public VectorStoreObservationContext.Builder createObservationContextBuilder(String operationName) { + + return VectorStoreObservationContext.builder(VectorStoreProvider.SIMPLE.value(), operationName) + .withDimensions(this.embeddingModel.dimensions()) + .withCollectionName("in-memory-map") + .withSimilarityMetric(VectorStoreSimilarityMetric.COSINE.value()); + } + + public static final class EmbeddingMath { + + private EmbeddingMath() { + throw new UnsupportedOperationException("This is a utility class and cannot be instantiated"); + } + + public static double cosineSimilarity(float[] vectorX, float[] vectorY) { + if (vectorX == null || vectorY == null) { + throw new RuntimeException("Vectors must not be null"); + } + if (vectorX.length != vectorY.length) { + throw new IllegalArgumentException("Vectors lengths must be equal"); + } + + float dotProduct = dotProduct(vectorX, vectorY); + float normX = norm(vectorX); + float normY = norm(vectorY); + + if (normX == 0 || normY == 0) { + throw new IllegalArgumentException("Vectors cannot have zero norm"); + } + + return dotProduct / (Math.sqrt(normX) * Math.sqrt(normY)); + } + + public static float dotProduct(float[] vectorX, float[] vectorY) { + if (vectorX.length != vectorY.length) { + throw new IllegalArgumentException("Vectors lengths must be equal"); + } + + float result = 0; + for (int i = 0; i < vectorX.length; ++i) { + result += vectorX[i] * vectorY[i]; + } + + return result; + } + + public static float norm(float[] vector) { + return dotProduct(vector, vector); + } + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStoreContent.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStoreContent.java new file mode 100644 index 000000000..b44389678 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStoreContent.java @@ -0,0 +1,180 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import org.springframework.ai.document.Document; +import org.springframework.ai.document.DocumentMetadata; +import org.springframework.ai.document.id.IdGenerator; +import org.springframework.ai.document.id.RandomIdGenerator; +import org.springframework.ai.model.Content; +import org.springframework.util.Assert; + +/** + * An immutable {@link Content} implementation representing content, metadata, and its + * embeddings. This class is thread-safe and all its fields are final and deeply + * immutable. The embedding vector is required for all instances of this class. + */ +public final class SimpleVectorStoreContent implements Content { + + private final String id; + + private final String text; + + private final Map metadata; + + private final float[] embedding; + + /** + * Creates a new instance with the given content, empty metadata, and embedding + * vector. + * @param text the content text, must not be null + * @param embedding the embedding vector, must not be null + */ + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public SimpleVectorStoreContent(@JsonProperty("text") @JsonAlias({ "content" }) String text, + @JsonProperty("embedding") float[] embedding) { + this(text, new HashMap<>(), embedding); + } + + /** + * Creates a new instance with the given content, metadata, and embedding vector. + * @param text the content text, must not be null + * @param metadata the metadata map, must not be null + * @param embedding the embedding vector, must not be null + */ + public SimpleVectorStoreContent(String text, Map metadata, float[] embedding) { + this(text, metadata, new RandomIdGenerator(), embedding); + } + + /** + * Creates a new instance with the given content, metadata, custom ID generator, and + * embedding vector. + * @param text the content text, must not be null + * @param metadata the metadata map, must not be null + * @param idGenerator the ID generator to use, must not be null + * @param embedding the embedding vector, must not be null + */ + public SimpleVectorStoreContent(String text, Map metadata, IdGenerator idGenerator, + float[] embedding) { + this(idGenerator.generateId(text, metadata), text, metadata, embedding); + } + + /** + * Creates a new instance with all fields specified. + * @param id the unique identifier, must not be empty + * @param text the content text, must not be null + * @param metadata the metadata map, must not be null + * @param embedding the embedding vector, must not be null + * @throws IllegalArgumentException if any parameter is null or if id is empty + */ + public SimpleVectorStoreContent(String id, String text, Map metadata, float[] embedding) { + Assert.hasText(id, "id must not be null or empty"); + Assert.notNull(text, "content must not be null"); + Assert.notNull(metadata, "metadata must not be null"); + Assert.notNull(embedding, "embedding must not be null"); + Assert.isTrue(embedding.length > 0, "embedding vector must not be empty"); + + this.id = id; + this.text = text; + this.metadata = Collections.unmodifiableMap(new HashMap<>(metadata)); + this.embedding = Arrays.copyOf(embedding, embedding.length); + } + + /** + * Creates a new instance with an updated embedding vector. + * @param embedding the new embedding vector, must not be null + * @return a new instance with the updated embedding + * @throws IllegalArgumentException if embedding is null or empty + */ + public SimpleVectorStoreContent withEmbedding(float[] embedding) { + Assert.notNull(embedding, "embedding must not be null"); + Assert.isTrue(embedding.length > 0, "embedding vector must not be empty"); + return new SimpleVectorStoreContent(this.id, this.text, this.metadata, embedding); + } + + public String getId() { + return this.id; + } + + @Override + public String getText() { + return this.text; + } + + @Override + public String getContent() { + return this.text; + } + + @Override + public Map getMetadata() { + return this.metadata; + } + + /** + * Returns a defensive copy of the embedding vector. + * @return a new array containing the embedding vector + */ + public float[] getEmbedding() { + return Arrays.copyOf(this.embedding, this.embedding.length); + } + + public Document toDocument(Double score) { + var metadata = new HashMap<>(this.metadata); + metadata.put(DocumentMetadata.DISTANCE.value(), 1.0 - score); + return Document.builder().id(this.id).text(this.text).metadata(metadata).score(score).build(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SimpleVectorStoreContent that = (SimpleVectorStoreContent) o; + return Objects.equals(this.id, that.id) && Objects.equals(this.text, that.text) + && Objects.equals(this.metadata, that.metadata) && Arrays.equals(this.embedding, that.embedding); + } + + @Override + public int hashCode() { + int result = Objects.hashCode(this.id); + result = 31 * result + Objects.hashCode(this.text); + result = 31 * result + Objects.hashCode(this.metadata); + result = 31 * result + Arrays.hashCode(this.embedding); + return result; + } + + @Override + public String toString() { + return "SimpleVectorStoreContent{" + "id='" + this.id + '\'' + ", content='" + this.text + '\'' + ", metadata=" + + this.metadata + ", embedding=" + Arrays.toString(this.embedding) + '}'; + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/VectorStore.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/VectorStore.java new file mode 100644 index 000000000..faea243a7 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/VectorStore.java @@ -0,0 +1,119 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore; + +import java.util.List; +import java.util.Optional; + +import io.micrometer.observation.ObservationRegistry; + +import org.springframework.ai.document.Document; +import org.springframework.ai.document.DocumentWriter; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.observation.DefaultVectorStoreObservationConvention; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention; +import org.springframework.lang.Nullable; + +/** + * The {@code VectorStore} interface defines the operations for managing and querying + * documents in a vector database. It extends {@link DocumentWriter} to support document + * writing operations. Vector databases are specialized for AI applications, performing + * similarity searches based on vector representations of data rather than exact matches. + * This interface allows for adding, deleting, and searching documents based on their + * similarity to a given query. + */ +public interface VectorStore extends DocumentWriter { + + default String getName() { + return this.getClass().getSimpleName(); + } + + /** + * Adds list of {@link Document}s to the vector store. + * @param documents the list of documents to store. Throws an exception if the + * underlying provider checks for duplicate IDs. + */ + void add(List documents); + + @Override + default void accept(List documents) { + add(documents); + } + + /** + * Deletes documents from the vector store. + * @param idList list of document ids for which documents will be removed. + * @return Returns true if the documents were successfully deleted. + */ + Optional delete(List idList); + + /** + * Retrieves documents by query embedding similarity and metadata filters to retrieve + * exactly the number of nearest-neighbor results that match the request criteria. + * @param request Search request for set search parameters, such as the query text, + * topK, similarity threshold and metadata filter expressions. + * @return Returns documents th match the query request conditions. + */ + List similaritySearch(SearchRequest request); + + /** + * Retrieves documents by query embedding similarity using the default + * {@link SearchRequest}'s' search criteria. + * @param query Text to use for embedding similarity comparison. + * @return Returns a list of documents that have embeddings similar to the query text + * embedding. + */ + default List similaritySearch(String query) { + return this.similaritySearch(SearchRequest.query(query)); + } + + /** + * Builder interface for creating VectorStore instances. Implements a fluent builder + * pattern for configuring observation-related settings. + * + * @param the concrete builder type, enabling method chaining with the correct + * return type + */ + interface Builder> { + + T embeddingModel(EmbeddingModel embeddingModel); + + /** + * Sets the registry for collecting observations and metrics. Defaults to + * {@link ObservationRegistry#NOOP} if not specified. + * @param observationRegistry the registry to use for observations + * @return the builder instance for method chaining + */ + T observationRegistry(ObservationRegistry observationRegistry); + + /** + * Sets a custom convention for creating observations. If not specified, + * {@link DefaultVectorStoreObservationConvention} will be used. + * @param convention the custom observation convention to use + * @return the builder instance for method chaining + */ + T customObservationConvention(VectorStoreObservationConvention convention); + + /** + * Builds and returns a new VectorStore instance with the configured settings. + * @return a new VectorStore instance + */ + VectorStore build(); + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/Filter.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/Filter.java new file mode 100644 index 000000000..53e16c691 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/Filter.java @@ -0,0 +1,143 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +/** + * Portable runtime generative for metadata filter expressions. This generic generative is + * used to define store agnostic filter expressions than later can be converted into + * vector-store specific, native, expressions. + * + * The expression generative supports constant comparison + * {@code (e.g. ==, !=, <, <=, >, >=) }, IN/NON-IN checks and AND and OR to compose + * multiple expressions. + * + * For example: + * + * 
{@code
+ * // 1: country == "BG"
+ * new Expression(EQ, new Key("country"), new Value("BG"));
+ *
+ * // 2: genre == "drama" AND year >= 2020
+ * new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")),
+ * 		new Expression(GTE, new Key("year"), new Value(2020)));
+ *
+ * // 3: genre in ["comedy", "documentary", "drama"]
+ * new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama")));
+ *
+ * // 4: year >= 2020 OR country == "BG" AND city != "Sofia"
+ * new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)),
+ * 		new Expression(AND, new Expression(EQ, new Key("country"), new Value("BG")),
+ * 				new Expression(NE, new Key("city"), new Value("Sofia"))));
+ *
+ * // 5: (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"]
+ * new Expression(AND,
+ * 		new Group(new Expression(OR, new Expression(EQ, new Key("country"), new Value("BG")),
+ * 				new Expression(GTE, new Key("year"), new Value(2020)))),
+ * 		new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Varna"))));
+ *
+ * // 6: isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]
+ * new Expression(AND, new Expression(EQ, new Key("isOpen"), new Value(true)),
+ * 		new Expression(AND, new Expression(GTE, new Key("year"), new Value(2020)),
+ * 				new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))));
+ *
+ * }
+ * + * + * Usually you will not create expression manually but use either the + * {@link FilterExpressionBuilder} DSL or the {@link FilterExpressionTextParser} for + * parsing generic text expressions. + * + * @author Christian Tzolov + */ +public class Filter { + + /** + * Filter expression operations.
+ * + * - EQ, NE, GT, GTE, LT, LTE operations supports "Key ExprType Value" + * expressions.
+ * + * - AND, OR are binary operations that support "(Expression|Group) ExprType + * (Expression|Group)" expressions.
+ * + * - IN, NIN support "Key (IN|NIN) ArrayValue" expression.
+ */ + public enum ExpressionType { + + AND, OR, EQ, NE, GT, GTE, LT, LTE, IN, NIN, NOT + + } + + /** + * Mark interface representing the supported expression types: {@link Key}, + * {@link Value}, {@link Expression} and {@link Group}. + */ + public interface Operand { + + } + + /** + * String identifier representing an expression key. (e.g. the country in the country + * == "NL" expression). + * + * @param key expression key + */ + public record Key(String key) implements Operand { + + } + + /** + * Represents expression value constant or constant array. Support Numeric, Boolean + * and String data types. + * + * @param value value constant or constant array + */ + public record Value(Object value) implements Operand { + + } + + /** + * Triple that represents and filter boolean expression as + * left type right. + * + * @param type Specify the expression type. + * @param left For comparison and inclusion expression types, the operand must be of + * type {@link Key} and for the AND|OR expression types the left operand must be + * another {@link Expression}. + * @param right For comparison and inclusion expression types, the operand must be of + * type {@link Value} or array of values. For the AND|OR type the right operand must + * be another {@link Expression}. + */ + public record Expression(ExpressionType type, Operand left, Operand right) implements Operand { + + public Expression(ExpressionType type, Operand operand) { + this(type, operand, null); + } + + } + + /** + * Represents expression grouping (e.g. (...) ) that indicates that the group needs to + * be evaluated with a precedence. + * + * @param content Inner expression to be evaluated as a part of the group. + */ + public record Group(Expression content) implements Operand { + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilder.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilder.java new file mode 100644 index 000000000..f7410c898 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionBuilder.java @@ -0,0 +1,130 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +import java.util.List; + +import org.springframework.ai.vectorstore.filter.Filter.ExpressionType; +import org.springframework.ai.vectorstore.filter.Filter.Key; +import org.springframework.ai.vectorstore.filter.Filter.Value; + +/** + * DSL builder for {@link Filter.Expression} instances. Here are some common examples: + * + * 
{@code
+ * var b = new FilterExpressionBuilder();
+ *
+ * // 1: country == "BG"
+ * var exp1 = b.eq("country", "BG");
+ *
+ * // 2: genre == "drama" AND year >= 2020
+ * var exp2 = b.and(b.eq("genre", "drama"), b.gte("year", 2020));
+ *
+ * // 3: genre in ["comedy", "documentary", "drama"]
+ * var exp3 = b.in("genre", "comedy", "documentary", "drama");
+ *
+ * // 4: year >= 2020 OR country == "BG" AND city != "Sofia"
+ * var exp4 = b.and(b.or(b.gte("year", 2020), b.eq("country", "BG")), b.ne("city", "Sofia"));
+ *
+ * // 5: (year >= 2020 OR country == "BG") AND city NIN ["Sofia", "Plovdiv"]
+ * var exp5 = b.and(b.group(b.or(b.gte("year", 2020), b.eq("country", "BG"))), b.nin("city", "Sofia", "Plovdiv"));
+ *
+ * // 6: isOpen == true AND year >= 2020 AND country IN ["BG", "NL", "US"]
+ * var exp6 = b.and(b.and(b.eq("isOpen", true), b.gte("year", 2020)), b.in("country", "BG", "NL", "US"));
+ *
+ * }
+ * + * + * This builder DSL mimics the common https://www.baeldung.com/hibernate-criteria-queries + * syntax. + * + * @author Christian Tzolov + */ +public class FilterExpressionBuilder { + + public Op eq(String key, Object value) { + return new Op(new Filter.Expression(ExpressionType.EQ, new Key(key), new Value(value))); + } + + public Op ne(String key, Object value) { + return new Op(new Filter.Expression(ExpressionType.NE, new Key(key), new Value(value))); + } + + public Op gt(String key, Object value) { + return new Op(new Filter.Expression(ExpressionType.GT, new Key(key), new Value(value))); + } + + public Op gte(String key, Object value) { + return new Op(new Filter.Expression(ExpressionType.GTE, new Key(key), new Value(value))); + } + + public Op lt(String key, Object value) { + return new Op(new Filter.Expression(ExpressionType.LT, new Key(key), new Value(value))); + } + + public Op lte(String key, Object value) { + return new Op(new Filter.Expression(ExpressionType.LTE, new Key(key), new Value(value))); + } + + public Op and(Op left, Op right) { + return new Op(new Filter.Expression(ExpressionType.AND, left.expression, right.expression)); + } + + public Op or(Op left, Op right) { + return new Op(new Filter.Expression(ExpressionType.OR, left.expression, right.expression)); + } + + public Op in(String key, Object... values) { + return this.in(key, List.of(values)); + } + + public Op in(String key, List values) { + return new Op(new Filter.Expression(ExpressionType.IN, new Key(key), new Value(values))); + } + + public Op nin(String key, Object... values) { + return this.nin(key, List.of(values)); + } + + public Op nin(String key, List values) { + return new Op(new Filter.Expression(ExpressionType.NIN, new Key(key), new Value(values))); + } + + public Op group(Op content) { + return new Op(new Filter.Group(content.build())); + } + + public Op not(Op content) { + return new Op(new Filter.Expression(ExpressionType.NOT, content.expression, null)); + } + + public record Op(Filter.Operand expression) { + + public Filter.Expression build() { + if (this.expression instanceof Filter.Group group) { + // Remove the top-level grouping. + return group.content(); + } + else if (this.expression instanceof Filter.Expression exp) { + return exp; + } + throw new RuntimeException("Invalid expression: " + this.expression); + } + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionConverter.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionConverter.java new file mode 100644 index 000000000..26d322ad6 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionConverter.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +/** + * Converters a generic, portable {@link Filter.Expression} into a + * {@link org.springframework.ai.vectorstore.VectorStore} specific expression language + * format. + * + * @author Christian Tzolov + */ +public interface FilterExpressionConverter { + + /** + * Convert the given {@link Filter.Expression} into a {@link String} representation. + * @param expression the expression to convert + * @return the converted expression + */ + String convertExpression(Filter.Expression expression); + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParser.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParser.java new file mode 100644 index 000000000..7d5e332c2 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParser.java @@ -0,0 +1,303 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; + +import org.antlr.v4.runtime.ANTLRErrorStrategy; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.misc.ParseCancellationException; + +import org.springframework.ai.vectorstore.filter.antlr4.FiltersBaseVisitor; +import org.springframework.ai.vectorstore.filter.antlr4.FiltersLexer; +import org.springframework.ai.vectorstore.filter.antlr4.FiltersParser; +import org.springframework.ai.vectorstore.filter.antlr4.FiltersParser.NotExpressionContext; +import org.springframework.core.NestedExceptionUtils; +import org.springframework.util.Assert; + +/** + * + * Parse a textual, vector-store agnostic, filter expression language into + * {@link Filter.Expression}. + * + * The vector-store agnostic, filter expression language is defined by a formal ANTLR4 + * grammar (Filters.g4). The language looks and feels like a subset of the well known SQL + * WHERE filter expressions. For example you can use the parser like this: + * + * 
{@code
+ *
+ * var parser = new FilterExpressionTextParser();
+ *
+ * exp1 = parser.parse("country == 'BG'"); // creates:
+ *  |
+ *  +->	new Expression(EQ, new Key("country"), new Value("BG"));
+ *
+ * exp2 = parser.parse("genre == 'drama' && year >= 2020"); // creates:
+ *  |
+ *  +->	new Expression(AND,
+ * 			new Expression(EQ, new Key("genre"), new Value("drama")),
+ * 			new Expression(GTE, new Key("year"), new Value(2020)));
+ *
+ * exp3 = parser.parse("genre in ['comedy', 'documentary', 'drama']");
+ *  |
+ *  +->	new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama")));
+ *
+ * exp4 = parser.parse("year >= 2020 || country == 'BG' && city != 'Sofia'");
+ *  |
+ *  +->	new Expression(OR,
+ * 			new Expression(GTE, new Key("year"), new Value(2020)),
+ * 			new Expression(AND,
+ * 					new Expression(EQ, new Key("country"), new Value("BG")),
+ * 					new Expression(NE, new Key("city"), new Value("Sofia"))));
+ *
+ * exp5 = parser.parse("(year >= 2020 || country == \"BG\") && city NOT IN ['Sofia', \"Plovdiv\"]"); // creates:
+ *  |
+ *  +->	new Expression(AND,
+ * 			new Group(new Expression(OR, new Expression(EQ, new Key("country"), new Value("BG")),
+ * 				new Expression(GTE, new Key("year"), new Value(2020)))),
+ * 			new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Varna"))));
+ *
+ * exp6 = parser.parse("isOpen == true && year >= 2020 && country IN ['BG', 'NL', 'US']"); // creates:
+ *  |
+ *  +->	new Expression(AND,
+ * 			new Expression(EQ, new Key("isOpen"), new Value(true)),
+ * 			new Expression(AND,
+ * 				new Expression(GTE, new Key("year"), new Value(2020)),
+ * 				new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US")))));
+ *
+ * exp7 = parser.parse("price >= 15.6 && price <= 20.13"); // creates:
+ *  |
+ *  +->	new Expression(AND,
+ * 			new Expression(GTE, new Key("price"), new Value(15.6)),
+ * 			new Expression(LTE, new Key("price"), new Value(20.13)));
+ *
+ * }
+ * + * @author Christian Tzolov + */ +public class FilterExpressionTextParser { + + private static final String WHERE_PREFIX = "WHERE"; + + private final DescriptiveErrorListener errorListener; + + private final ANTLRErrorStrategy errorHandler; + + private final Map cache = new ConcurrentHashMap<>(); + + public FilterExpressionTextParser() { + this(new BailErrorStrategy()); + } + + public FilterExpressionTextParser(ANTLRErrorStrategy handler) { + this.errorListener = DescriptiveErrorListener.INSTANCE; + this.errorHandler = handler; + } + + public Filter.Expression parse(String textFilterExpression) { + + Assert.hasText(textFilterExpression, "Expression should not be empty!"); + + // Prefix the expression with the compulsory WHERE keyword. + if (!textFilterExpression.toUpperCase().startsWith(WHERE_PREFIX)) { + textFilterExpression = String.format("%s %s", WHERE_PREFIX, textFilterExpression); + } + + if (this.cache.containsKey(textFilterExpression)) { + return this.cache.get(textFilterExpression); + } + + var lexer = new FiltersLexer(CharStreams.fromString(textFilterExpression)); + var tokens = new CommonTokenStream(lexer); + var parser = new FiltersParser(tokens); + + parser.removeErrorListeners(); + this.errorListener.errorMessages.clear(); + parser.addErrorListener(this.errorListener); + + if (this.errorHandler != null) { + parser.setErrorHandler(this.errorHandler); + } + + var filterExpressionVisitor = new FilterExpressionVisitor(); + try { + Filter.Operand operand = filterExpressionVisitor.visit(parser.where()); + var filterExpression = filterExpressionVisitor.castToExpression(operand); + this.cache.putIfAbsent(textFilterExpression, filterExpression); + return filterExpression; + } + catch (ParseCancellationException e) { + var msg = this.errorListener.errorMessages.stream().collect(Collectors.joining()); + var rootCause = NestedExceptionUtils.getRootCause(e); + throw new FilterExpressionParseException(msg, rootCause); + } + } + + public void clearCache() { + this.cache.clear(); + } + + /** For testing only */ + Map getCache() { + return this.cache; + } + + public static class FilterExpressionParseException extends RuntimeException { + + public FilterExpressionParseException(String message, Throwable cause) { + super(message, cause); + } + + } + + public static class FilterExpressionVisitor extends FiltersBaseVisitor { + + private static final Map COMP_EXPRESSION_TYPE_MAP = Map.of("==", + Filter.ExpressionType.EQ, "!=", Filter.ExpressionType.NE, ">", Filter.ExpressionType.GT, ">=", + Filter.ExpressionType.GTE, "<", Filter.ExpressionType.LT, "<=", Filter.ExpressionType.LTE); + + @Override + public Filter.Operand visitWhere(FiltersParser.WhereContext ctx) { + return this.visit(ctx.booleanExpression()); + } + + @Override + public Filter.Operand visitIdentifier(FiltersParser.IdentifierContext ctx) { + return new Filter.Key(ctx.getText()); + } + + @Override + public Filter.Operand visitTextConstant(FiltersParser.TextConstantContext ctx) { + String onceQuotedText = removeOuterQuotes(ctx.getText()); + return new Filter.Value(onceQuotedText); + } + + private String removeOuterQuotes(String in) { + return in.substring(1, in.length() - 1); + } + + @Override + public Filter.Operand visitIntegerConstant(FiltersParser.IntegerConstantContext ctx) { + return new Filter.Value(Integer.valueOf(ctx.getText())); + } + + @Override + public Filter.Operand visitDecimalConstant(FiltersParser.DecimalConstantContext ctx) { + return new Filter.Value(Double.valueOf(ctx.getText())); + } + + @Override + public Filter.Operand visitBooleanConstant(FiltersParser.BooleanConstantContext ctx) { + return new Filter.Value(Boolean.valueOf(ctx.getText())); + } + + @Override + public Filter.Operand visitConstantArray(FiltersParser.ConstantArrayContext ctx) { + List list = new ArrayList<>(); + ctx.constant().forEach(constantCtx -> list.add(((Filter.Value) this.visit(constantCtx)).value())); + return new Filter.Value(list); + } + + @Override + public Filter.Operand visitInExpression(FiltersParser.InExpressionContext ctx) { + return new Filter.Expression(Filter.ExpressionType.IN, this.visitIdentifier(ctx.identifier()), + this.visitConstantArray(ctx.constantArray())); + } + + @Override + public Filter.Operand visitNinExpression(FiltersParser.NinExpressionContext ctx) { + return new Filter.Expression(Filter.ExpressionType.NIN, this.visitIdentifier(ctx.identifier()), + this.visitConstantArray(ctx.constantArray())); + } + + @Override + public Filter.Operand visitCompareExpression(FiltersParser.CompareExpressionContext ctx) { + return new Filter.Expression(this.covertCompare(ctx.compare().getText()), + this.visitIdentifier(ctx.identifier()), this.visit(ctx.constant())); + } + + private Filter.ExpressionType covertCompare(String compare) { + if (!COMP_EXPRESSION_TYPE_MAP.containsKey(compare)) { + throw new RuntimeException("Unknown compare operator: " + compare); + } + return COMP_EXPRESSION_TYPE_MAP.get(compare); + } + + @Override + public Filter.Operand visitAndExpression(FiltersParser.AndExpressionContext ctx) { + return new Filter.Expression(Filter.ExpressionType.AND, this.visit(ctx.left), this.visit(ctx.right)); + } + + @Override + public Filter.Operand visitOrExpression(FiltersParser.OrExpressionContext ctx) { + return new Filter.Expression(Filter.ExpressionType.OR, this.visit(ctx.left), this.visit(ctx.right)); + } + + @Override + public Filter.Operand visitGroupExpression(FiltersParser.GroupExpressionContext ctx) { + return new Filter.Group(castToExpression(this.visit(ctx.booleanExpression()))); + } + + @Override + public Filter.Operand visitNotExpression(NotExpressionContext ctx) { + return new Filter.Expression(Filter.ExpressionType.NOT, this.visit(ctx.booleanExpression()), null); + } + + public Filter.Expression castToExpression(Filter.Operand expression) { + if (expression instanceof Filter.Group group) { + // Remove the top-level grouping. + return group.content(); + } + else if (expression instanceof Filter.Expression exp) { + return exp; + } + throw new RuntimeException("Invalid expression: " + expression); + } + + } + + public static class DescriptiveErrorListener extends BaseErrorListener { + + public static final DescriptiveErrorListener INSTANCE = new DescriptiveErrorListener(); + + public final List errorMessages = new CopyOnWriteArrayList<>(); + + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, + String msg, RecognitionException e) { + + String sourceName = recognizer.getInputStream().getSourceName(); + + var errorMessage = String.format("Source: %s, Line: %s:%s, Error: %s", sourceName, line, charPositionInLine, + msg); + + this.errorMessages.add(errorMessage); + } + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterHelper.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterHelper.java new file mode 100644 index 000000000..ce2bebf91 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterHelper.java @@ -0,0 +1,204 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.springframework.ai.vectorstore.filter.Filter.Expression; +import org.springframework.ai.vectorstore.filter.Filter.ExpressionType; +import org.springframework.ai.vectorstore.filter.Filter.Operand; +import org.springframework.util.Assert; + +/** + * Helper class providing various boolean transformation. + * + * @author Christian Tzolov + */ +public final class FilterHelper { + + private final static Map TYPE_NEGATION_MAP = Map.of(ExpressionType.AND, + ExpressionType.OR, ExpressionType.OR, ExpressionType.AND, ExpressionType.EQ, ExpressionType.NE, + ExpressionType.NE, ExpressionType.EQ, ExpressionType.GT, ExpressionType.LTE, ExpressionType.GTE, + ExpressionType.LT, ExpressionType.LT, ExpressionType.GTE, ExpressionType.LTE, ExpressionType.GT, + ExpressionType.IN, ExpressionType.NIN, ExpressionType.NIN, ExpressionType.IN); + + private FilterHelper() { + } + + /** + * Transforms the input expression into a semantically equivalent one with negation + * operators propagated thought the expression tree by following the negation rules: + * + * 
+	 * 	NOT(NOT(a)) = a
+	 *
+	 * 	NOT(a AND b) = NOT(a) OR NOT(b)
+	 * 	NOT(a OR b) = NOT(a) AND NOT(b)
+	 *
+	 * 	NOT(a EQ b) = a NE b
+	 * 	NOT(a NE b) = a EQ b
+	 *
+	 * 	NOT(a GT b) = a LTE b
+	 * 	NOT(a GTE b) = a LT b
+	 *
+	 * 	NOT(a LT b) = a GTE b
+	 * 	NOT(a LTE b) = a GT b
+	 *
+	 * 	NOT(a IN [...]) = a NIN [...]
+	 * 	NOT(a NIN [...]) = a IN [...]
+	 *
+ * @param operand Filter expression to negate. + * @return Returns an negation of the input expression. + */ + public static Filter.Operand negate(Filter.Operand operand) { + + if (operand instanceof Filter.Group group) { + Operand inEx = negate(group.content()); + if (inEx instanceof Filter.Group inEx2) { + inEx = inEx2.content(); + } + return new Filter.Group((Expression) inEx); + } + else if (operand instanceof Filter.Expression exp) { + switch (exp.type()) { + case NOT: // NOT(NOT(a)) = a + return negate(exp.left()); + case AND: // NOT(a AND b) = NOT(a) OR NOT(b) + case OR: // NOT(a OR b) = NOT(a) AND NOT(b) + return new Filter.Expression(TYPE_NEGATION_MAP.get(exp.type()), negate(exp.left()), + negate(exp.right())); + case EQ: // NOT(e EQ b) = e NE b + case NE: // NOT(e NE b) = e EQ b + case GT: // NOT(e GT b) = e LTE b + case GTE: // NOT(e GTE b) = e LT b + case LT: // NOT(e LT b) = e GTE b + case LTE: // NOT(e LTE b) = e GT b + return new Filter.Expression(TYPE_NEGATION_MAP.get(exp.type()), exp.left(), exp.right()); + case IN: // NOT(e IN [...]) = e NIN [...] + case NIN: // NOT(e NIN [...]) = e IN [...] + return new Filter.Expression(TYPE_NEGATION_MAP.get(exp.type()), exp.left(), exp.right()); + default: + throw new IllegalArgumentException("Unknown expression type: " + exp.type()); + } + } + else { + throw new IllegalArgumentException("Can not negate operand of type: " + operand.getClass()); + } + } + + /** + * Expands the IN into a semantically equivalent boolean expressions of ORs of EQs. + * Useful for providers that don't provide native IN support. + * + * For example the 
+	 * foo IN ["bar1", "bar2", "bar3"]
+	 *
+ * + * expression is equivalent to + * + * 
+	 * {@code foo == "bar1" || foo == "bar2" || foo == "bar3" (e.g. OR(foo EQ "bar1" OR(foo EQ "bar2" OR(foo EQ "bar3")))}
+	 *
+ * @param exp input IN expression. + * @param context Output native expression. + * @param filterExpressionConverter {@link FilterExpressionConverter} used to compose + * the OR and EQ expanded expressions. + */ + public static void expandIn(Expression exp, StringBuilder context, + FilterExpressionConverter filterExpressionConverter) { + Assert.isTrue(exp.type() == ExpressionType.IN, "Expected IN expressions but was: " + exp.type()); + expandInNinExpressions(ExpressionType.OR, ExpressionType.EQ, exp, context, filterExpressionConverter); + } + + /** + * + * Expands the NIN (e.g. NOT IN) into a semantically equivalent boolean expressions of + * ANDs of NEs. Useful for providers that don't provide native NIN support.
+ * + * For example the + * + * 
+	 * foo NIN ["bar1", "bar2", "bar3"] (or foo NOT IN ["bar1", "bar2", "bar3"])
+	 *
+ * + * express is equivalent to + * + * 
+	 * {@code foo != "bar1" && foo != "bar2" && foo != "bar3" (e.g. AND(foo NE "bar1" AND( foo NE "bar2" OR(foo NE "bar3"))) )}
+	 *
+ * @param exp input NIN expression. + * @param context Output native expression. + * @param filterExpressionConverter {@link FilterExpressionConverter} used to compose + * the AND and NE expanded expressions. + */ + public static void expandNin(Expression exp, StringBuilder context, + FilterExpressionConverter filterExpressionConverter) { + Assert.isTrue(exp.type() == ExpressionType.NIN, "Expected NIN expressions but was: " + exp.type()); + expandInNinExpressions(ExpressionType.AND, ExpressionType.NE, exp, context, filterExpressionConverter); + } + + private static void expandInNinExpressions(Filter.ExpressionType outerExpressionType, + Filter.ExpressionType innerExpressionType, Expression exp, StringBuilder context, + FilterExpressionConverter expressionConverter) { + if (exp.right() instanceof Filter.Value value) { + if (value.value() instanceof List list) { + // 1. foo IN ["bar1", "bar2", "bar3"] is equivalent to foo == "bar1" || + // foo == "bar2" || foo == "bar3" + // or equivalent to OR(foo == "bar1" OR( foo == "bar2" OR(foo == "bar3"))) + // 2. foo IN ["bar1", "bar2", "bar3"] is equivalent to foo != "bar1" && + // foo != "bar2" && foo != "bar3" + // or equivalent to AND(foo != "bar1" AND( foo != "bar2" OR(foo != + // "bar3"))) + List eqExprs = new ArrayList<>(); + for (Object o : list) { + eqExprs.add(new Filter.Expression(innerExpressionType, exp.left(), new Filter.Value(o))); + } + context.append(expressionConverter.convertExpression(aggregate(outerExpressionType, eqExprs))); + } + else { + // 1. foo IN ["bar"] is equivalent to foo == "BAR" + // 2. foo NIN ["bar"] is equivalent to foo != "BAR" + context.append(expressionConverter + .convertExpression(new Filter.Expression(innerExpressionType, exp.left(), exp.right()))); + } + } + else { + throw new IllegalStateException( + "Filter IN right expression should be of Filter.Value type but was " + exp.right().getClass()); + } + } + + /** + * Recursively aggregates a list of expression into a binary tree with 'aggregateType' + * join nodes. + * @param aggregateType type all tree splits. + * @param expressions list of expressions to aggregate. + * @return Returns a binary tree expression. + */ + private static Filter.Expression aggregate(Filter.ExpressionType aggregateType, + List expressions) { + + if (expressions.size() == 1) { + return expressions.get(0); + } + return new Filter.Expression(aggregateType, expressions.get(0), + aggregate(aggregateType, expressions.subList(1, expressions.size()))); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/Filters.interp b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/Filters.interp new file mode 100644 index 000000000..51775a8a5 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/Filters.interp @@ -0,0 +1,69 @@ +token literal names: +null +null +'.' +',' +'[' +']' +'(' +')' +'==' +'-' +'+' +'>' +'>=' +'<' +'<=' +'!=' +null +null +null +null +null +null +null +null +null +null +null + +token symbolic names: +null +WHERE +DOT +COMMA +LEFT_SQUARE_BRACKETS +RIGHT_SQUARE_BRACKETS +LEFT_PARENTHESIS +RIGHT_PARENTHESIS +EQUALS +MINUS +PLUS +GT +GE +LT +LE +NE +AND +OR +IN +NIN +NOT +BOOLEAN_VALUE +QUOTED_STRING +INTEGER_VALUE +DECIMAL_VALUE +IDENTIFIER +WS + +rule names: +where +booleanExpression +constantArray +compare +identifier +constant + + +atn: +[4, 1, 26, 89, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 30, 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 40, 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 48, 8, 1, 10, 1, 12, 1, 51, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 5, 2, 57, 8, 2, 10, 2, 12, 2, 60, 9, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 71, 8, 4, 1, 5, 3, 5, 74, 8, 5, 1, 5, 1, 5, 3, 5, 78, 8, 5, 1, 5, 1, 5, 4, 5, 82, 8, 5, 11, 5, 12, 5, 83, 1, 5, 3, 5, 87, 8, 5, 1, 5, 0, 1, 2, 6, 0, 2, 4, 6, 8, 10, 0, 2, 2, 0, 8, 8, 11, 15, 1, 0, 9, 10, 98, 0, 12, 1, 0, 0, 0, 2, 39, 1, 0, 0, 0, 4, 52, 1, 0, 0, 0, 6, 63, 1, 0, 0, 0, 8, 70, 1, 0, 0, 0, 10, 86, 1, 0, 0, 0, 12, 13, 5, 1, 0, 0, 13, 14, 3, 2, 1, 0, 14, 15, 5, 0, 0, 1, 15, 1, 1, 0, 0, 0, 16, 17, 6, 1, -1, 0, 17, 18, 3, 8, 4, 0, 18, 19, 3, 6, 3, 0, 19, 20, 3, 10, 5, 0, 20, 40, 1, 0, 0, 0, 21, 22, 3, 8, 4, 0, 22, 23, 5, 18, 0, 0, 23, 24, 3, 4, 2, 0, 24, 40, 1, 0, 0, 0, 25, 29, 3, 8, 4, 0, 26, 27, 5, 20, 0, 0, 27, 30, 5, 18, 0, 0, 28, 30, 5, 19, 0, 0, 29, 26, 1, 0, 0, 0, 29, 28, 1, 0, 0, 0, 30, 31, 1, 0, 0, 0, 31, 32, 3, 4, 2, 0, 32, 40, 1, 0, 0, 0, 33, 34, 5, 6, 0, 0, 34, 35, 3, 2, 1, 0, 35, 36, 5, 7, 0, 0, 36, 40, 1, 0, 0, 0, 37, 38, 5, 20, 0, 0, 38, 40, 3, 2, 1, 1, 39, 16, 1, 0, 0, 0, 39, 21, 1, 0, 0, 0, 39, 25, 1, 0, 0, 0, 39, 33, 1, 0, 0, 0, 39, 37, 1, 0, 0, 0, 40, 49, 1, 0, 0, 0, 41, 42, 10, 4, 0, 0, 42, 43, 5, 16, 0, 0, 43, 48, 3, 2, 1, 5, 44, 45, 10, 3, 0, 0, 45, 46, 5, 17, 0, 0, 46, 48, 3, 2, 1, 4, 47, 41, 1, 0, 0, 0, 47, 44, 1, 0, 0, 0, 48, 51, 1, 0, 0, 0, 49, 47, 1, 0, 0, 0, 49, 50, 1, 0, 0, 0, 50, 3, 1, 0, 0, 0, 51, 49, 1, 0, 0, 0, 52, 53, 5, 4, 0, 0, 53, 58, 3, 10, 5, 0, 54, 55, 5, 3, 0, 0, 55, 57, 3, 10, 5, 0, 56, 54, 1, 0, 0, 0, 57, 60, 1, 0, 0, 0, 58, 56, 1, 0, 0, 0, 58, 59, 1, 0, 0, 0, 59, 61, 1, 0, 0, 0, 60, 58, 1, 0, 0, 0, 61, 62, 5, 5, 0, 0, 62, 5, 1, 0, 0, 0, 63, 64, 7, 0, 0, 0, 64, 7, 1, 0, 0, 0, 65, 66, 5, 25, 0, 0, 66, 67, 5, 2, 0, 0, 67, 71, 5, 25, 0, 0, 68, 71, 5, 25, 0, 0, 69, 71, 5, 22, 0, 0, 70, 65, 1, 0, 0, 0, 70, 68, 1, 0, 0, 0, 70, 69, 1, 0, 0, 0, 71, 9, 1, 0, 0, 0, 72, 74, 7, 1, 0, 0, 73, 72, 1, 0, 0, 0, 73, 74, 1, 0, 0, 0, 74, 75, 1, 0, 0, 0, 75, 87, 5, 23, 0, 0, 76, 78, 7, 1, 0, 0, 77, 76, 1, 0, 0, 0, 77, 78, 1, 0, 0, 0, 78, 79, 1, 0, 0, 0, 79, 87, 5, 24, 0, 0, 80, 82, 5, 22, 0, 0, 81, 80, 1, 0, 0, 0, 82, 83, 1, 0, 0, 0, 83, 81, 1, 0, 0, 0, 83, 84, 1, 0, 0, 0, 84, 87, 1, 0, 0, 0, 85, 87, 5, 21, 0, 0, 86, 73, 1, 0, 0, 0, 86, 77, 1, 0, 0, 0, 86, 81, 1, 0, 0, 0, 86, 85, 1, 0, 0, 0, 87, 11, 1, 0, 0, 0, 10, 29, 39, 47, 49, 58, 70, 73, 77, 83, 86] \ No newline at end of file diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersBaseListener.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersBaseListener.java new file mode 100644 index 000000000..136aedbab --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersBaseListener.java @@ -0,0 +1,411 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.antlr4; + +// Generated from org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 by ANTLR 4.13.1 + +// ############################################################ +// # NOTE: This is ANTLR4 auto-generated code. Do not modify! # +// ############################################################ + +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.TerminalNode; + +/** + * This class provides an empty implementation of {@link FiltersListener}, which can be + * extended to create a listener which only needs to handle a subset of the available + * methods. + */ +@SuppressWarnings("CheckReturnValue") +public class FiltersBaseListener implements FiltersListener { + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterWhere(FiltersParser.WhereContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitWhere(FiltersParser.WhereContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterNinExpression(FiltersParser.NinExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitNinExpression(FiltersParser.NinExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterAndExpression(FiltersParser.AndExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitAndExpression(FiltersParser.AndExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterInExpression(FiltersParser.InExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitInExpression(FiltersParser.InExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterNotExpression(FiltersParser.NotExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitNotExpression(FiltersParser.NotExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterCompareExpression(FiltersParser.CompareExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitCompareExpression(FiltersParser.CompareExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterOrExpression(FiltersParser.OrExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitOrExpression(FiltersParser.OrExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterGroupExpression(FiltersParser.GroupExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitGroupExpression(FiltersParser.GroupExpressionContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterConstantArray(FiltersParser.ConstantArrayContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitConstantArray(FiltersParser.ConstantArrayContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterCompare(FiltersParser.CompareContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitCompare(FiltersParser.CompareContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterIdentifier(FiltersParser.IdentifierContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitIdentifier(FiltersParser.IdentifierContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterIntegerConstant(FiltersParser.IntegerConstantContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitIntegerConstant(FiltersParser.IntegerConstantContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterDecimalConstant(FiltersParser.DecimalConstantContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitDecimalConstant(FiltersParser.DecimalConstantContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterTextConstant(FiltersParser.TextConstantContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitTextConstant(FiltersParser.TextConstantContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterBooleanConstant(FiltersParser.BooleanConstantContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitBooleanConstant(FiltersParser.BooleanConstantContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void enterEveryRule(ParserRuleContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void exitEveryRule(ParserRuleContext ctx) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void visitTerminal(TerminalNode node) { + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation does nothing. + *

+ */ + @Override + public void visitErrorNode(ErrorNode node) { + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersBaseVisitor.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersBaseVisitor.java new file mode 100644 index 000000000..f8a5a2041 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersBaseVisitor.java @@ -0,0 +1,233 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.antlr4; + +// Generated from org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 by ANTLR 4.13.1 + +// ############################################################ +// # NOTE: This is ANTLR4 auto-generated code. Do not modify! # +// ############################################################ + +import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor; + +/** + * This class provides an empty implementation of {@link FiltersVisitor}, which can be + * extended to create a visitor which only needs to handle a subset of the available + * methods. + * + * @param The return type of the visit operation. Use {@link Void} for operations with + * no return type. + */ +@SuppressWarnings("CheckReturnValue") +public class FiltersBaseVisitor extends AbstractParseTreeVisitor implements FiltersVisitor { + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitWhere(FiltersParser.WhereContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitNinExpression(FiltersParser.NinExpressionContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitAndExpression(FiltersParser.AndExpressionContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitInExpression(FiltersParser.InExpressionContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitNotExpression(FiltersParser.NotExpressionContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitCompareExpression(FiltersParser.CompareExpressionContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitOrExpression(FiltersParser.OrExpressionContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitGroupExpression(FiltersParser.GroupExpressionContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitConstantArray(FiltersParser.ConstantArrayContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitCompare(FiltersParser.CompareContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitIdentifier(FiltersParser.IdentifierContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitIntegerConstant(FiltersParser.IntegerConstantContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitDecimalConstant(FiltersParser.DecimalConstantContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitTextConstant(FiltersParser.TextConstantContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + * + *

+ * The default implementation returns the result of calling {@link #visitChildren} on + * {@code ctx}. + *

+ */ + @Override + public T visitBooleanConstant(FiltersParser.BooleanConstantContext ctx) { + return visitChildren(ctx); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersLexer.interp b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersLexer.interp new file mode 100644 index 000000000..919669898 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersLexer.interp @@ -0,0 +1,98 @@ +token literal names: +null +null +'.' +',' +'[' +']' +'(' +')' +'==' +'-' +'+' +'>' +'>=' +'<' +'<=' +'!=' +null +null +null +null +null +null +null +null +null +null +null + +token symbolic names: +null +WHERE +DOT +COMMA +LEFT_SQUARE_BRACKETS +RIGHT_SQUARE_BRACKETS +LEFT_PARENTHESIS +RIGHT_PARENTHESIS +EQUALS +MINUS +PLUS +GT +GE +LT +LE +NE +AND +OR +IN +NIN +NOT +BOOLEAN_VALUE +QUOTED_STRING +INTEGER_VALUE +DECIMAL_VALUE +IDENTIFIER +WS + +rule names: +WHERE +DOT +COMMA +LEFT_SQUARE_BRACKETS +RIGHT_SQUARE_BRACKETS +LEFT_PARENTHESIS +RIGHT_PARENTHESIS +EQUALS +MINUS +PLUS +GT +GE +LT +LE +NE +AND +OR +IN +NIN +NOT +BOOLEAN_VALUE +QUOTED_STRING +INTEGER_VALUE +DECIMAL_VALUE +IDENTIFIER +DECIMAL_DIGITS +DIGIT +LETTER +WS + +channel names: +DEFAULT_TOKEN_CHANNEL +HIDDEN + +mode names: +DEFAULT_MODE + +atn: +[4, 0, 26, 230, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 3, 0, 70, 8, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 9, 1, 9, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 112, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 3, 16, 120, 8, 16, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 126, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 134, 8, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 3, 19, 142, 8, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 3, 20, 162, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 168, 8, 21, 10, 21, 12, 21, 171, 9, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 178, 8, 21, 10, 21, 12, 21, 181, 9, 21, 1, 21, 3, 21, 184, 8, 21, 1, 22, 4, 22, 187, 8, 22, 11, 22, 12, 22, 188, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 4, 24, 196, 8, 24, 11, 24, 12, 24, 197, 1, 25, 4, 25, 201, 8, 25, 11, 25, 12, 25, 202, 1, 25, 1, 25, 5, 25, 207, 8, 25, 10, 25, 12, 25, 210, 9, 25, 1, 25, 1, 25, 4, 25, 214, 8, 25, 11, 25, 12, 25, 215, 3, 25, 218, 8, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 4, 28, 225, 8, 28, 11, 28, 12, 28, 226, 1, 28, 1, 28, 0, 0, 29, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 25, 13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 20, 41, 21, 43, 22, 45, 23, 47, 24, 49, 25, 51, 0, 53, 0, 55, 0, 57, 26, 1, 0, 5, 2, 0, 39, 39, 92, 92, 2, 0, 34, 34, 92, 92, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 3, 0, 9, 10, 13, 13, 32, 32, 251, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 1, 69, 1, 0, 0, 0, 3, 71, 1, 0, 0, 0, 5, 73, 1, 0, 0, 0, 7, 75, 1, 0, 0, 0, 9, 77, 1, 0, 0, 0, 11, 79, 1, 0, 0, 0, 13, 81, 1, 0, 0, 0, 15, 83, 1, 0, 0, 0, 17, 86, 1, 0, 0, 0, 19, 88, 1, 0, 0, 0, 21, 90, 1, 0, 0, 0, 23, 92, 1, 0, 0, 0, 25, 95, 1, 0, 0, 0, 27, 97, 1, 0, 0, 0, 29, 100, 1, 0, 0, 0, 31, 111, 1, 0, 0, 0, 33, 119, 1, 0, 0, 0, 35, 125, 1, 0, 0, 0, 37, 133, 1, 0, 0, 0, 39, 141, 1, 0, 0, 0, 41, 161, 1, 0, 0, 0, 43, 183, 1, 0, 0, 0, 45, 186, 1, 0, 0, 0, 47, 190, 1, 0, 0, 0, 49, 195, 1, 0, 0, 0, 51, 217, 1, 0, 0, 0, 53, 219, 1, 0, 0, 0, 55, 221, 1, 0, 0, 0, 57, 224, 1, 0, 0, 0, 59, 60, 5, 87, 0, 0, 60, 61, 5, 72, 0, 0, 61, 62, 5, 69, 0, 0, 62, 63, 5, 82, 0, 0, 63, 70, 5, 69, 0, 0, 64, 65, 5, 119, 0, 0, 65, 66, 5, 104, 0, 0, 66, 67, 5, 101, 0, 0, 67, 68, 5, 114, 0, 0, 68, 70, 5, 101, 0, 0, 69, 59, 1, 0, 0, 0, 69, 64, 1, 0, 0, 0, 70, 2, 1, 0, 0, 0, 71, 72, 5, 46, 0, 0, 72, 4, 1, 0, 0, 0, 73, 74, 5, 44, 0, 0, 74, 6, 1, 0, 0, 0, 75, 76, 5, 91, 0, 0, 76, 8, 1, 0, 0, 0, 77, 78, 5, 93, 0, 0, 78, 10, 1, 0, 0, 0, 79, 80, 5, 40, 0, 0, 80, 12, 1, 0, 0, 0, 81, 82, 5, 41, 0, 0, 82, 14, 1, 0, 0, 0, 83, 84, 5, 61, 0, 0, 84, 85, 5, 61, 0, 0, 85, 16, 1, 0, 0, 0, 86, 87, 5, 45, 0, 0, 87, 18, 1, 0, 0, 0, 88, 89, 5, 43, 0, 0, 89, 20, 1, 0, 0, 0, 90, 91, 5, 62, 0, 0, 91, 22, 1, 0, 0, 0, 92, 93, 5, 62, 0, 0, 93, 94, 5, 61, 0, 0, 94, 24, 1, 0, 0, 0, 95, 96, 5, 60, 0, 0, 96, 26, 1, 0, 0, 0, 97, 98, 5, 60, 0, 0, 98, 99, 5, 61, 0, 0, 99, 28, 1, 0, 0, 0, 100, 101, 5, 33, 0, 0, 101, 102, 5, 61, 0, 0, 102, 30, 1, 0, 0, 0, 103, 104, 5, 65, 0, 0, 104, 105, 5, 78, 0, 0, 105, 112, 5, 68, 0, 0, 106, 107, 5, 97, 0, 0, 107, 108, 5, 110, 0, 0, 108, 112, 5, 100, 0, 0, 109, 110, 5, 38, 0, 0, 110, 112, 5, 38, 0, 0, 111, 103, 1, 0, 0, 0, 111, 106, 1, 0, 0, 0, 111, 109, 1, 0, 0, 0, 112, 32, 1, 0, 0, 0, 113, 114, 5, 79, 0, 0, 114, 120, 5, 82, 0, 0, 115, 116, 5, 111, 0, 0, 116, 120, 5, 114, 0, 0, 117, 118, 5, 124, 0, 0, 118, 120, 5, 124, 0, 0, 119, 113, 1, 0, 0, 0, 119, 115, 1, 0, 0, 0, 119, 117, 1, 0, 0, 0, 120, 34, 1, 0, 0, 0, 121, 122, 5, 73, 0, 0, 122, 126, 5, 78, 0, 0, 123, 124, 5, 105, 0, 0, 124, 126, 5, 110, 0, 0, 125, 121, 1, 0, 0, 0, 125, 123, 1, 0, 0, 0, 126, 36, 1, 0, 0, 0, 127, 128, 5, 78, 0, 0, 128, 129, 5, 73, 0, 0, 129, 134, 5, 78, 0, 0, 130, 131, 5, 110, 0, 0, 131, 132, 5, 105, 0, 0, 132, 134, 5, 110, 0, 0, 133, 127, 1, 0, 0, 0, 133, 130, 1, 0, 0, 0, 134, 38, 1, 0, 0, 0, 135, 136, 5, 78, 0, 0, 136, 137, 5, 79, 0, 0, 137, 142, 5, 84, 0, 0, 138, 139, 5, 110, 0, 0, 139, 140, 5, 111, 0, 0, 140, 142, 5, 116, 0, 0, 141, 135, 1, 0, 0, 0, 141, 138, 1, 0, 0, 0, 142, 40, 1, 0, 0, 0, 143, 144, 5, 84, 0, 0, 144, 145, 5, 82, 0, 0, 145, 146, 5, 85, 0, 0, 146, 162, 5, 69, 0, 0, 147, 148, 5, 116, 0, 0, 148, 149, 5, 114, 0, 0, 149, 150, 5, 117, 0, 0, 150, 162, 5, 101, 0, 0, 151, 152, 5, 70, 0, 0, 152, 153, 5, 65, 0, 0, 153, 154, 5, 76, 0, 0, 154, 155, 5, 83, 0, 0, 155, 162, 5, 69, 0, 0, 156, 157, 5, 102, 0, 0, 157, 158, 5, 97, 0, 0, 158, 159, 5, 108, 0, 0, 159, 160, 5, 115, 0, 0, 160, 162, 5, 101, 0, 0, 161, 143, 1, 0, 0, 0, 161, 147, 1, 0, 0, 0, 161, 151, 1, 0, 0, 0, 161, 156, 1, 0, 0, 0, 162, 42, 1, 0, 0, 0, 163, 169, 5, 39, 0, 0, 164, 168, 8, 0, 0, 0, 165, 166, 5, 92, 0, 0, 166, 168, 9, 0, 0, 0, 167, 164, 1, 0, 0, 0, 167, 165, 1, 0, 0, 0, 168, 171, 1, 0, 0, 0, 169, 167, 1, 0, 0, 0, 169, 170, 1, 0, 0, 0, 170, 172, 1, 0, 0, 0, 171, 169, 1, 0, 0, 0, 172, 184, 5, 39, 0, 0, 173, 179, 5, 34, 0, 0, 174, 178, 8, 1, 0, 0, 175, 176, 5, 92, 0, 0, 176, 178, 9, 0, 0, 0, 177, 174, 1, 0, 0, 0, 177, 175, 1, 0, 0, 0, 178, 181, 1, 0, 0, 0, 179, 177, 1, 0, 0, 0, 179, 180, 1, 0, 0, 0, 180, 182, 1, 0, 0, 0, 181, 179, 1, 0, 0, 0, 182, 184, 5, 34, 0, 0, 183, 163, 1, 0, 0, 0, 183, 173, 1, 0, 0, 0, 184, 44, 1, 0, 0, 0, 185, 187, 3, 53, 26, 0, 186, 185, 1, 0, 0, 0, 187, 188, 1, 0, 0, 0, 188, 186, 1, 0, 0, 0, 188, 189, 1, 0, 0, 0, 189, 46, 1, 0, 0, 0, 190, 191, 3, 51, 25, 0, 191, 48, 1, 0, 0, 0, 192, 196, 3, 55, 27, 0, 193, 196, 3, 53, 26, 0, 194, 196, 5, 95, 0, 0, 195, 192, 1, 0, 0, 0, 195, 193, 1, 0, 0, 0, 195, 194, 1, 0, 0, 0, 196, 197, 1, 0, 0, 0, 197, 195, 1, 0, 0, 0, 197, 198, 1, 0, 0, 0, 198, 50, 1, 0, 0, 0, 199, 201, 3, 53, 26, 0, 200, 199, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 200, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 204, 1, 0, 0, 0, 204, 208, 5, 46, 0, 0, 205, 207, 3, 53, 26, 0, 206, 205, 1, 0, 0, 0, 207, 210, 1, 0, 0, 0, 208, 206, 1, 0, 0, 0, 208, 209, 1, 0, 0, 0, 209, 218, 1, 0, 0, 0, 210, 208, 1, 0, 0, 0, 211, 213, 5, 46, 0, 0, 212, 214, 3, 53, 26, 0, 213, 212, 1, 0, 0, 0, 214, 215, 1, 0, 0, 0, 215, 213, 1, 0, 0, 0, 215, 216, 1, 0, 0, 0, 216, 218, 1, 0, 0, 0, 217, 200, 1, 0, 0, 0, 217, 211, 1, 0, 0, 0, 218, 52, 1, 0, 0, 0, 219, 220, 7, 2, 0, 0, 220, 54, 1, 0, 0, 0, 221, 222, 7, 3, 0, 0, 222, 56, 1, 0, 0, 0, 223, 225, 7, 4, 0, 0, 224, 223, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 224, 1, 0, 0, 0, 226, 227, 1, 0, 0, 0, 227, 228, 1, 0, 0, 0, 228, 229, 6, 28, 0, 0, 229, 58, 1, 0, 0, 0, 21, 0, 69, 111, 119, 125, 133, 141, 161, 167, 169, 177, 179, 183, 188, 195, 197, 202, 208, 215, 217, 226, 1, 0, 1, 0] \ No newline at end of file diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersLexer.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersLexer.java new file mode 100644 index 000000000..dc1491d54 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersLexer.java @@ -0,0 +1,311 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.antlr4; + +// Generated from org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 by ANTLR 4.13.1 + +// ############################################################ +// # NOTE: This is ANTLR4 auto-generated code. Do not modify! # +// ############################################################ + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.RuntimeMetaData; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.atn.LexerATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.dfa.DFA; + +@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue", "this-escape" }) +public class FiltersLexer extends Lexer { + + public static final int WHERE = 1, DOT = 2, COMMA = 3, LEFT_SQUARE_BRACKETS = 4, RIGHT_SQUARE_BRACKETS = 5, + LEFT_PARENTHESIS = 6, RIGHT_PARENTHESIS = 7, EQUALS = 8, MINUS = 9, PLUS = 10, GT = 11, GE = 12, LT = 13, + LE = 14, NE = 15, AND = 16, OR = 17, IN = 18, NIN = 19, NOT = 20, BOOLEAN_VALUE = 21, QUOTED_STRING = 22, + INTEGER_VALUE = 23, DECIMAL_VALUE = 24, IDENTIFIER = 25, WS = 26; + + public static final String[] ruleNames = makeRuleNames(); + + /** + * @deprecated Use {@link #VOCABULARY} instead. + */ + @Deprecated + public static final String[] tokenNames; + + public static final String _serializedATN = "\u0004\u0000\u001a\u00e6\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002" + + "\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002" + + "\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002" + + "\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002" + + "\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e" + + "\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011" + + "\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014" + + "\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017" + + "\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a" + + "\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0001\u0000\u0001\u0000" + + "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000" + + "\u0001\u0000\u0001\u0000\u0003\u0000F\b\u0000\u0001\u0001\u0001\u0001" + + "\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001\u0004\u0001\u0004" + + "\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001\u0007\u0001\u0007" + + "\u0001\u0007\u0001\b\u0001\b\u0001\t\u0001\t\u0001\n\u0001\n\u0001\u000b" + + "\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001" + + "\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001" + + "\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0003\u000fp\b" + + "\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001" + + "\u0010\u0003\u0010x\b\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001" + + "\u0011\u0003\u0011~\b\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001" + + "\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u0086\b\u0012\u0001\u0013\u0001" + + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0003\u0013\u008e" + + "\b\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001" + + "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001" + + "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001" + + "\u0014\u0003\u0014\u00a2\b\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001" + + "\u0015\u0005\u0015\u00a8\b\u0015\n\u0015\f\u0015\u00ab\t\u0015\u0001\u0015" + + "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u00b2\b\u0015" + + "\n\u0015\f\u0015\u00b5\t\u0015\u0001\u0015\u0003\u0015\u00b8\b\u0015\u0001" + + "\u0016\u0004\u0016\u00bb\b\u0016\u000b\u0016\f\u0016\u00bc\u0001\u0017" + + "\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0004\u0018\u00c4\b\u0018" + + "\u000b\u0018\f\u0018\u00c5\u0001\u0019\u0004\u0019\u00c9\b\u0019\u000b" + + "\u0019\f\u0019\u00ca\u0001\u0019\u0001\u0019\u0005\u0019\u00cf\b\u0019" + + "\n\u0019\f\u0019\u00d2\t\u0019\u0001\u0019\u0001\u0019\u0004\u0019\u00d6" + + "\b\u0019\u000b\u0019\f\u0019\u00d7\u0003\u0019\u00da\b\u0019\u0001\u001a" + + "\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001c\u0004\u001c\u00e1\b\u001c" + + "\u000b\u001c\f\u001c\u00e2\u0001\u001c\u0001\u001c\u0000\u0000\u001d\u0001" + + "\u0001\u0003\u0002\u0005\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007" + + "\u000f\b\u0011\t\u0013\n\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d" + + "\u000f\u001f\u0010!\u0011#\u0012%\u0013\'\u0014)\u0015+\u0016-\u0017/" + + "\u00181\u00193\u00005\u00007\u00009\u001a\u0001\u0000\u0005\u0002\u0000" + + "\'\'\\\\\u0002\u0000\"\"\\\\\u0001\u000009\u0002\u0000AZaz\u0003\u0000" + + "\t\n\r\r \u00fb\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001" + + "\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001" + + "\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000" + + "\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000" + + "\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000" + + "\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000" + + "\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000" + + "\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000" + + "\u0000\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000" + + "%\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001" + + "\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000" + + "\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u0000" + + "9\u0001\u0000\u0000\u0000\u0001E\u0001\u0000\u0000\u0000\u0003G\u0001" + + "\u0000\u0000\u0000\u0005I\u0001\u0000\u0000\u0000\u0007K\u0001\u0000\u0000" + + "\u0000\tM\u0001\u0000\u0000\u0000\u000bO\u0001\u0000\u0000\u0000\rQ\u0001" + + "\u0000\u0000\u0000\u000fS\u0001\u0000\u0000\u0000\u0011V\u0001\u0000\u0000" + + "\u0000\u0013X\u0001\u0000\u0000\u0000\u0015Z\u0001\u0000\u0000\u0000\u0017" + + "\\\u0001\u0000\u0000\u0000\u0019_\u0001\u0000\u0000\u0000\u001ba\u0001" + + "\u0000\u0000\u0000\u001dd\u0001\u0000\u0000\u0000\u001fo\u0001\u0000\u0000" + + "\u0000!w\u0001\u0000\u0000\u0000#}\u0001\u0000\u0000\u0000%\u0085\u0001" + + "\u0000\u0000\u0000\'\u008d\u0001\u0000\u0000\u0000)\u00a1\u0001\u0000" + + "\u0000\u0000+\u00b7\u0001\u0000\u0000\u0000-\u00ba\u0001\u0000\u0000\u0000" + + "/\u00be\u0001\u0000\u0000\u00001\u00c3\u0001\u0000\u0000\u00003\u00d9" + + "\u0001\u0000\u0000\u00005\u00db\u0001\u0000\u0000\u00007\u00dd\u0001\u0000" + + "\u0000\u00009\u00e0\u0001\u0000\u0000\u0000;<\u0005W\u0000\u0000<=\u0005" + + "H\u0000\u0000=>\u0005E\u0000\u0000>?\u0005R\u0000\u0000?F\u0005E\u0000" + + "\u0000@A\u0005w\u0000\u0000AB\u0005h\u0000\u0000BC\u0005e\u0000\u0000" + + "CD\u0005r\u0000\u0000DF\u0005e\u0000\u0000E;\u0001\u0000\u0000\u0000E" + + "@\u0001\u0000\u0000\u0000F\u0002\u0001\u0000\u0000\u0000GH\u0005.\u0000" + + "\u0000H\u0004\u0001\u0000\u0000\u0000IJ\u0005,\u0000\u0000J\u0006\u0001" + + "\u0000\u0000\u0000KL\u0005[\u0000\u0000L\b\u0001\u0000\u0000\u0000MN\u0005" + + "]\u0000\u0000N\n\u0001\u0000\u0000\u0000OP\u0005(\u0000\u0000P\f\u0001" + + "\u0000\u0000\u0000QR\u0005)\u0000\u0000R\u000e\u0001\u0000\u0000\u0000" + + "ST\u0005=\u0000\u0000TU\u0005=\u0000\u0000U\u0010\u0001\u0000\u0000\u0000" + + "VW\u0005-\u0000\u0000W\u0012\u0001\u0000\u0000\u0000XY\u0005+\u0000\u0000" + + "Y\u0014\u0001\u0000\u0000\u0000Z[\u0005>\u0000\u0000[\u0016\u0001\u0000" + + "\u0000\u0000\\]\u0005>\u0000\u0000]^\u0005=\u0000\u0000^\u0018\u0001\u0000" + + "\u0000\u0000_`\u0005<\u0000\u0000`\u001a\u0001\u0000\u0000\u0000ab\u0005" + + "<\u0000\u0000bc\u0005=\u0000\u0000c\u001c\u0001\u0000\u0000\u0000de\u0005" + + "!\u0000\u0000ef\u0005=\u0000\u0000f\u001e\u0001\u0000\u0000\u0000gh\u0005" + + "A\u0000\u0000hi\u0005N\u0000\u0000ip\u0005D\u0000\u0000jk\u0005a\u0000" + + "\u0000kl\u0005n\u0000\u0000lp\u0005d\u0000\u0000mn\u0005&\u0000\u0000" + + "np\u0005&\u0000\u0000og\u0001\u0000\u0000\u0000oj\u0001\u0000\u0000\u0000" + + "om\u0001\u0000\u0000\u0000p \u0001\u0000\u0000\u0000qr\u0005O\u0000\u0000" + + "rx\u0005R\u0000\u0000st\u0005o\u0000\u0000tx\u0005r\u0000\u0000uv\u0005" + + "|\u0000\u0000vx\u0005|\u0000\u0000wq\u0001\u0000\u0000\u0000ws\u0001\u0000" + + "\u0000\u0000wu\u0001\u0000\u0000\u0000x\"\u0001\u0000\u0000\u0000yz\u0005" + + "I\u0000\u0000z~\u0005N\u0000\u0000{|\u0005i\u0000\u0000|~\u0005n\u0000" + + "\u0000}y\u0001\u0000\u0000\u0000}{\u0001\u0000\u0000\u0000~$\u0001\u0000" + + "\u0000\u0000\u007f\u0080\u0005N\u0000\u0000\u0080\u0081\u0005I\u0000\u0000" + + "\u0081\u0086\u0005N\u0000\u0000\u0082\u0083\u0005n\u0000\u0000\u0083\u0084" + + "\u0005i\u0000\u0000\u0084\u0086\u0005n\u0000\u0000\u0085\u007f\u0001\u0000" + + "\u0000\u0000\u0085\u0082\u0001\u0000\u0000\u0000\u0086&\u0001\u0000\u0000" + + "\u0000\u0087\u0088\u0005N\u0000\u0000\u0088\u0089\u0005O\u0000\u0000\u0089" + + "\u008e\u0005T\u0000\u0000\u008a\u008b\u0005n\u0000\u0000\u008b\u008c\u0005" + + "o\u0000\u0000\u008c\u008e\u0005t\u0000\u0000\u008d\u0087\u0001\u0000\u0000" + + "\u0000\u008d\u008a\u0001\u0000\u0000\u0000\u008e(\u0001\u0000\u0000\u0000" + + "\u008f\u0090\u0005T\u0000\u0000\u0090\u0091\u0005R\u0000\u0000\u0091\u0092" + + "\u0005U\u0000\u0000\u0092\u00a2\u0005E\u0000\u0000\u0093\u0094\u0005t" + + "\u0000\u0000\u0094\u0095\u0005r\u0000\u0000\u0095\u0096\u0005u\u0000\u0000" + + "\u0096\u00a2\u0005e\u0000\u0000\u0097\u0098\u0005F\u0000\u0000\u0098\u0099" + + "\u0005A\u0000\u0000\u0099\u009a\u0005L\u0000\u0000\u009a\u009b\u0005S" + + "\u0000\u0000\u009b\u00a2\u0005E\u0000\u0000\u009c\u009d\u0005f\u0000\u0000" + + "\u009d\u009e\u0005a\u0000\u0000\u009e\u009f\u0005l\u0000\u0000\u009f\u00a0" + + "\u0005s\u0000\u0000\u00a0\u00a2\u0005e\u0000\u0000\u00a1\u008f\u0001\u0000" + + "\u0000\u0000\u00a1\u0093\u0001\u0000\u0000\u0000\u00a1\u0097\u0001\u0000" + + "\u0000\u0000\u00a1\u009c\u0001\u0000\u0000\u0000\u00a2*\u0001\u0000\u0000" + + "\u0000\u00a3\u00a9\u0005\'\u0000\u0000\u00a4\u00a8\b\u0000\u0000\u0000" + + "\u00a5\u00a6\u0005\\\u0000\u0000\u00a6\u00a8\t\u0000\u0000\u0000\u00a7" + + "\u00a4\u0001\u0000\u0000\u0000\u00a7\u00a5\u0001\u0000\u0000\u0000\u00a8" + + "\u00ab\u0001\u0000\u0000\u0000\u00a9\u00a7\u0001\u0000\u0000\u0000\u00a9" + + "\u00aa\u0001\u0000\u0000\u0000\u00aa\u00ac\u0001\u0000\u0000\u0000\u00ab" + + "\u00a9\u0001\u0000\u0000\u0000\u00ac\u00b8\u0005\'\u0000\u0000\u00ad\u00b3" + + "\u0005\"\u0000\u0000\u00ae\u00b2\b\u0001\u0000\u0000\u00af\u00b0\u0005" + + "\\\u0000\u0000\u00b0\u00b2\t\u0000\u0000\u0000\u00b1\u00ae\u0001\u0000" + + "\u0000\u0000\u00b1\u00af\u0001\u0000\u0000\u0000\u00b2\u00b5\u0001\u0000" + + "\u0000\u0000\u00b3\u00b1\u0001\u0000\u0000\u0000\u00b3\u00b4\u0001\u0000" + + "\u0000\u0000\u00b4\u00b6\u0001\u0000\u0000\u0000\u00b5\u00b3\u0001\u0000" + + "\u0000\u0000\u00b6\u00b8\u0005\"\u0000\u0000\u00b7\u00a3\u0001\u0000\u0000" + + "\u0000\u00b7\u00ad\u0001\u0000\u0000\u0000\u00b8,\u0001\u0000\u0000\u0000" + + "\u00b9\u00bb\u00035\u001a\u0000\u00ba\u00b9\u0001\u0000\u0000\u0000\u00bb" + + "\u00bc\u0001\u0000\u0000\u0000\u00bc\u00ba\u0001\u0000\u0000\u0000\u00bc" + + "\u00bd\u0001\u0000\u0000\u0000\u00bd.\u0001\u0000\u0000\u0000\u00be\u00bf" + + "\u00033\u0019\u0000\u00bf0\u0001\u0000\u0000\u0000\u00c0\u00c4\u00037" + + "\u001b\u0000\u00c1\u00c4\u00035\u001a\u0000\u00c2\u00c4\u0005_\u0000\u0000" + + "\u00c3\u00c0\u0001\u0000\u0000\u0000\u00c3\u00c1\u0001\u0000\u0000\u0000" + + "\u00c3\u00c2\u0001\u0000\u0000\u0000\u00c4\u00c5\u0001\u0000\u0000\u0000" + + "\u00c5\u00c3\u0001\u0000\u0000\u0000\u00c5\u00c6\u0001\u0000\u0000\u0000" + + "\u00c62\u0001\u0000\u0000\u0000\u00c7\u00c9\u00035\u001a\u0000\u00c8\u00c7" + + "\u0001\u0000\u0000\u0000\u00c9\u00ca\u0001\u0000\u0000\u0000\u00ca\u00c8" + + "\u0001\u0000\u0000\u0000\u00ca\u00cb\u0001\u0000\u0000\u0000\u00cb\u00cc" + + "\u0001\u0000\u0000\u0000\u00cc\u00d0\u0005.\u0000\u0000\u00cd\u00cf\u0003" + + "5\u001a\u0000\u00ce\u00cd\u0001\u0000\u0000\u0000\u00cf\u00d2\u0001\u0000" + + "\u0000\u0000\u00d0\u00ce\u0001\u0000\u0000\u0000\u00d0\u00d1\u0001\u0000" + + "\u0000\u0000\u00d1\u00da\u0001\u0000\u0000\u0000\u00d2\u00d0\u0001\u0000" + + "\u0000\u0000\u00d3\u00d5\u0005.\u0000\u0000\u00d4\u00d6\u00035\u001a\u0000" + + "\u00d5\u00d4\u0001\u0000\u0000\u0000\u00d6\u00d7\u0001\u0000\u0000\u0000" + + "\u00d7\u00d5\u0001\u0000\u0000\u0000\u00d7\u00d8\u0001\u0000\u0000\u0000" + + "\u00d8\u00da\u0001\u0000\u0000\u0000\u00d9\u00c8\u0001\u0000\u0000\u0000" + + "\u00d9\u00d3\u0001\u0000\u0000\u0000\u00da4\u0001\u0000\u0000\u0000\u00db" + + "\u00dc\u0007\u0002\u0000\u0000\u00dc6\u0001\u0000\u0000\u0000\u00dd\u00de" + + "\u0007\u0003\u0000\u0000\u00de8\u0001\u0000\u0000\u0000\u00df\u00e1\u0007" + + "\u0004\u0000\u0000\u00e0\u00df\u0001\u0000\u0000\u0000\u00e1\u00e2\u0001" + + "\u0000\u0000\u0000\u00e2\u00e0\u0001\u0000\u0000\u0000\u00e2\u00e3\u0001" + + "\u0000\u0000\u0000\u00e3\u00e4\u0001\u0000\u0000\u0000\u00e4\u00e5\u0006" + + "\u001c\u0000\u0000\u00e5:\u0001\u0000\u0000\u0000\u0015\u0000Eow}\u0085" + + "\u008d\u00a1\u00a7\u00a9\u00b1\u00b3\u00b7\u00bc\u00c3\u00c5\u00ca\u00d0" + + "\u00d7\u00d9\u00e2\u0001\u0000\u0001\u0000"; + + public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); + + protected static final DFA[] _decisionToDFA; + + protected static final PredictionContextCache _sharedContextCache = new PredictionContextCache(); + + private static final String[] _LITERAL_NAMES = makeLiteralNames(); + + private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); + + public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); + + public static String[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" }; + + public static String[] modeNames = { "DEFAULT_MODE" }; + + public FiltersLexer(CharStream input) { + super(input); + _interp = new LexerATNSimulator(this, _ATN, _decisionToDFA, _sharedContextCache); + } + + private static String[] makeRuleNames() { + return new String[] { "WHERE", "DOT", "COMMA", "LEFT_SQUARE_BRACKETS", "RIGHT_SQUARE_BRACKETS", + "LEFT_PARENTHESIS", "RIGHT_PARENTHESIS", "EQUALS", "MINUS", "PLUS", "GT", "GE", "LT", "LE", "NE", "AND", + "OR", "IN", "NIN", "NOT", "BOOLEAN_VALUE", "QUOTED_STRING", "INTEGER_VALUE", "DECIMAL_VALUE", + "IDENTIFIER", "DECIMAL_DIGITS", "DIGIT", "LETTER", "WS" }; + } + + private static String[] makeLiteralNames() { + return new String[] { null, null, "'.'", "','", "'['", "']'", "'('", "')'", "'=='", "'-'", "'+'", "'>'", "'>='", + "'<'", "'<='", "'!='" }; + } + + private static String[] makeSymbolicNames() { + return new String[] { null, "WHERE", "DOT", "COMMA", "LEFT_SQUARE_BRACKETS", "RIGHT_SQUARE_BRACKETS", + "LEFT_PARENTHESIS", "RIGHT_PARENTHESIS", "EQUALS", "MINUS", "PLUS", "GT", "GE", "LT", "LE", "NE", "AND", + "OR", "IN", "NIN", "NOT", "BOOLEAN_VALUE", "QUOTED_STRING", "INTEGER_VALUE", "DECIMAL_VALUE", + "IDENTIFIER", "WS" }; + } + + @Override + @Deprecated + public String[] getTokenNames() { + return tokenNames; + } + + @Override + + public Vocabulary getVocabulary() { + return VOCABULARY; + } + + @Override + public String getGrammarFileName() { + return "Filters.g4"; + } + + @Override + public String[] getRuleNames() { + return ruleNames; + } + + @Override + public String getSerializedATN() { + return _serializedATN; + } + + @Override + public String[] getChannelNames() { + return channelNames; + } + + @Override + public String[] getModeNames() { + return modeNames; + } + + @Override + public ATN getATN() { + return _ATN; + } + + static { + RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION); + } + + static { + tokenNames = new String[_SYMBOLIC_NAMES.length]; + for (int i = 0; i < tokenNames.length; i++) { + tokenNames[i] = VOCABULARY.getLiteralName(i); + if (tokenNames[i] == null) { + tokenNames[i] = VOCABULARY.getSymbolicName(i); + } + + if (tokenNames[i] == null) { + tokenNames[i] = ""; + } + } + } + + static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; + for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) { + _decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i); + } + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersListener.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersListener.java new file mode 100644 index 000000000..8e49aeff6 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersListener.java @@ -0,0 +1,235 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.antlr4; + +// Generated from org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 by ANTLR 4.13.1 + +// ############################################################ +// # NOTE: This is ANTLR4 auto-generated code. Do not modify! # +// ############################################################ + +import org.antlr.v4.runtime.tree.ParseTreeListener; + +/** + * This interface defines a complete listener for a parse tree produced by + * {@link FiltersParser}. + */ +public interface FiltersListener extends ParseTreeListener { + + /** + * Enter a parse tree produced by {@link FiltersParser#where}. + * @param ctx the parse tree + */ + void enterWhere(FiltersParser.WhereContext ctx); + + /** + * Exit a parse tree produced by {@link FiltersParser#where}. + * @param ctx the parse tree + */ + void exitWhere(FiltersParser.WhereContext ctx); + + /** + * Enter a parse tree produced by the {@code NinExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void enterNinExpression(FiltersParser.NinExpressionContext ctx); + + /** + * Exit a parse tree produced by the {@code NinExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void exitNinExpression(FiltersParser.NinExpressionContext ctx); + + /** + * Enter a parse tree produced by the {@code AndExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void enterAndExpression(FiltersParser.AndExpressionContext ctx); + + /** + * Exit a parse tree produced by the {@code AndExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void exitAndExpression(FiltersParser.AndExpressionContext ctx); + + /** + * Enter a parse tree produced by the {@code InExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void enterInExpression(FiltersParser.InExpressionContext ctx); + + /** + * Exit a parse tree produced by the {@code InExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void exitInExpression(FiltersParser.InExpressionContext ctx); + + /** + * Enter a parse tree produced by the {@code NotExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void enterNotExpression(FiltersParser.NotExpressionContext ctx); + + /** + * Exit a parse tree produced by the {@code NotExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void exitNotExpression(FiltersParser.NotExpressionContext ctx); + + /** + * Enter a parse tree produced by the {@code CompareExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void enterCompareExpression(FiltersParser.CompareExpressionContext ctx); + + /** + * Exit a parse tree produced by the {@code CompareExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void exitCompareExpression(FiltersParser.CompareExpressionContext ctx); + + /** + * Enter a parse tree produced by the {@code OrExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void enterOrExpression(FiltersParser.OrExpressionContext ctx); + + /** + * Exit a parse tree produced by the {@code OrExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void exitOrExpression(FiltersParser.OrExpressionContext ctx); + + /** + * Enter a parse tree produced by the {@code GroupExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void enterGroupExpression(FiltersParser.GroupExpressionContext ctx); + + /** + * Exit a parse tree produced by the {@code GroupExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + */ + void exitGroupExpression(FiltersParser.GroupExpressionContext ctx); + + /** + * Enter a parse tree produced by {@link FiltersParser#constantArray}. + * @param ctx the parse tree + */ + void enterConstantArray(FiltersParser.ConstantArrayContext ctx); + + /** + * Exit a parse tree produced by {@link FiltersParser#constantArray}. + * @param ctx the parse tree + */ + void exitConstantArray(FiltersParser.ConstantArrayContext ctx); + + /** + * Enter a parse tree produced by {@link FiltersParser#compare}. + * @param ctx the parse tree + */ + void enterCompare(FiltersParser.CompareContext ctx); + + /** + * Exit a parse tree produced by {@link FiltersParser#compare}. + * @param ctx the parse tree + */ + void exitCompare(FiltersParser.CompareContext ctx); + + /** + * Enter a parse tree produced by {@link FiltersParser#identifier}. + * @param ctx the parse tree + */ + void enterIdentifier(FiltersParser.IdentifierContext ctx); + + /** + * Exit a parse tree produced by {@link FiltersParser#identifier}. + * @param ctx the parse tree + */ + void exitIdentifier(FiltersParser.IdentifierContext ctx); + + /** + * Enter a parse tree produced by the {@code IntegerConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + */ + void enterIntegerConstant(FiltersParser.IntegerConstantContext ctx); + + /** + * Exit a parse tree produced by the {@code IntegerConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + */ + void exitIntegerConstant(FiltersParser.IntegerConstantContext ctx); + + /** + * Enter a parse tree produced by the {@code DecimalConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + */ + void enterDecimalConstant(FiltersParser.DecimalConstantContext ctx); + + /** + * Exit a parse tree produced by the {@code DecimalConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + */ + void exitDecimalConstant(FiltersParser.DecimalConstantContext ctx); + + /** + * Enter a parse tree produced by the {@code TextConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + */ + void enterTextConstant(FiltersParser.TextConstantContext ctx); + + /** + * Exit a parse tree produced by the {@code TextConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + */ + void exitTextConstant(FiltersParser.TextConstantContext ctx); + + /** + * Enter a parse tree produced by the {@code BooleanConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + */ + void enterBooleanConstant(FiltersParser.BooleanConstantContext ctx); + + /** + * Exit a parse tree produced by the {@code BooleanConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + */ + void exitBooleanConstant(FiltersParser.BooleanConstantContext ctx); + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersParser.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersParser.java new file mode 100644 index 000000000..945a3a953 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersParser.java @@ -0,0 +1,1406 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.antlr4; + +// Generated from org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 by ANTLR 4.13.1 + +// ############################################################ +// # NOTE: This is ANTLR4 auto-generated code. Do not modify! # +// ############################################################ + +import java.util.List; + +import org.antlr.v4.runtime.FailedPredicateException; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.RuntimeMetaData; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.ParseTreeVisitor; +import org.antlr.v4.runtime.tree.TerminalNode; + +@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" }) +public class FiltersParser extends Parser { + + public static final int WHERE = 1, DOT = 2, COMMA = 3, LEFT_SQUARE_BRACKETS = 4, RIGHT_SQUARE_BRACKETS = 5, + LEFT_PARENTHESIS = 6, RIGHT_PARENTHESIS = 7, EQUALS = 8, MINUS = 9, PLUS = 10, GT = 11, GE = 12, LT = 13, + LE = 14, NE = 15, AND = 16, OR = 17, IN = 18, NIN = 19, NOT = 20, BOOLEAN_VALUE = 21, QUOTED_STRING = 22, + INTEGER_VALUE = 23, DECIMAL_VALUE = 24, IDENTIFIER = 25, WS = 26; + + public static final int RULE_where = 0, RULE_booleanExpression = 1, RULE_constantArray = 2, RULE_compare = 3, + RULE_identifier = 4, RULE_constant = 5; + + public static final String[] ruleNames = makeRuleNames(); + + /** + * @deprecated Use {@link #VOCABULARY} instead. + */ + @Deprecated + public static final String[] tokenNames; + + public static final String _serializedATN = "\u0004\u0001\u001aY\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002" + + "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002" + + "\u0005\u0007\u0005\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001" + + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001" + + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001" + + "\u0001\u0003\u0001\u001e\b\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001" + + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0003\u0001(\b" + + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001" + + "\u0001\u0005\u00010\b\u0001\n\u0001\f\u00013\t\u0001\u0001\u0002\u0001" + + "\u0002\u0001\u0002\u0001\u0002\u0005\u00029\b\u0002\n\u0002\f\u0002<\t" + + "\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001\u0004\u0001" + + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0003\u0004G\b\u0004\u0001" + + "\u0005\u0003\u0005J\b\u0005\u0001\u0005\u0001\u0005\u0003\u0005N\b\u0005" + + "\u0001\u0005\u0001\u0005\u0004\u0005R\b\u0005\u000b\u0005\f\u0005S\u0001" + + "\u0005\u0003\u0005W\b\u0005\u0001\u0005\u0000\u0001\u0002\u0006\u0000" + + "\u0002\u0004\u0006\b\n\u0000\u0002\u0002\u0000\b\b\u000b\u000f\u0001\u0000" + + "\t\nb\u0000\f\u0001\u0000\u0000\u0000\u0002\'\u0001\u0000\u0000\u0000" + + "\u00044\u0001\u0000\u0000\u0000\u0006?\u0001\u0000\u0000\u0000\bF\u0001" + + "\u0000\u0000\u0000\nV\u0001\u0000\u0000\u0000\f\r\u0005\u0001\u0000\u0000" + + "\r\u000e\u0003\u0002\u0001\u0000\u000e\u000f\u0005\u0000\u0000\u0001\u000f" + + "\u0001\u0001\u0000\u0000\u0000\u0010\u0011\u0006\u0001\uffff\uffff\u0000" + + "\u0011\u0012\u0003\b\u0004\u0000\u0012\u0013\u0003\u0006\u0003\u0000\u0013" + + "\u0014\u0003\n\u0005\u0000\u0014(\u0001\u0000\u0000\u0000\u0015\u0016" + + "\u0003\b\u0004\u0000\u0016\u0017\u0005\u0012\u0000\u0000\u0017\u0018\u0003" + + "\u0004\u0002\u0000\u0018(\u0001\u0000\u0000\u0000\u0019\u001d\u0003\b" + + "\u0004\u0000\u001a\u001b\u0005\u0014\u0000\u0000\u001b\u001e\u0005\u0012" + + "\u0000\u0000\u001c\u001e\u0005\u0013\u0000\u0000\u001d\u001a\u0001\u0000" + + "\u0000\u0000\u001d\u001c\u0001\u0000\u0000\u0000\u001e\u001f\u0001\u0000" + + "\u0000\u0000\u001f \u0003\u0004\u0002\u0000 (\u0001\u0000\u0000\u0000" + + "!\"\u0005\u0006\u0000\u0000\"#\u0003\u0002\u0001\u0000#$\u0005\u0007\u0000" + + "\u0000$(\u0001\u0000\u0000\u0000%&\u0005\u0014\u0000\u0000&(\u0003\u0002" + + "\u0001\u0001\'\u0010\u0001\u0000\u0000\u0000\'\u0015\u0001\u0000\u0000" + + "\u0000\'\u0019\u0001\u0000\u0000\u0000\'!\u0001\u0000\u0000\u0000\'%\u0001" + + "\u0000\u0000\u0000(1\u0001\u0000\u0000\u0000)*\n\u0004\u0000\u0000*+\u0005" + + "\u0010\u0000\u0000+0\u0003\u0002\u0001\u0005,-\n\u0003\u0000\u0000-.\u0005" + + "\u0011\u0000\u0000.0\u0003\u0002\u0001\u0004/)\u0001\u0000\u0000\u0000" + + "/,\u0001\u0000\u0000\u000003\u0001\u0000\u0000\u00001/\u0001\u0000\u0000" + + "\u000012\u0001\u0000\u0000\u00002\u0003\u0001\u0000\u0000\u000031\u0001" + + "\u0000\u0000\u000045\u0005\u0004\u0000\u00005:\u0003\n\u0005\u000067\u0005" + + "\u0003\u0000\u000079\u0003\n\u0005\u000086\u0001\u0000\u0000\u00009<\u0001" + + "\u0000\u0000\u0000:8\u0001\u0000\u0000\u0000:;\u0001\u0000\u0000\u0000" + + ";=\u0001\u0000\u0000\u0000<:\u0001\u0000\u0000\u0000=>\u0005\u0005\u0000" + + "\u0000>\u0005\u0001\u0000\u0000\u0000?@\u0007\u0000\u0000\u0000@\u0007" + + "\u0001\u0000\u0000\u0000AB\u0005\u0019\u0000\u0000BC\u0005\u0002\u0000" + + "\u0000CG\u0005\u0019\u0000\u0000DG\u0005\u0019\u0000\u0000EG\u0005\u0016" + + "\u0000\u0000FA\u0001\u0000\u0000\u0000FD\u0001\u0000\u0000\u0000FE\u0001" + + "\u0000\u0000\u0000G\t\u0001\u0000\u0000\u0000HJ\u0007\u0001\u0000\u0000" + + "IH\u0001\u0000\u0000\u0000IJ\u0001\u0000\u0000\u0000JK\u0001\u0000\u0000" + + "\u0000KW\u0005\u0017\u0000\u0000LN\u0007\u0001\u0000\u0000ML\u0001\u0000" + + "\u0000\u0000MN\u0001\u0000\u0000\u0000NO\u0001\u0000\u0000\u0000OW\u0005" + + "\u0018\u0000\u0000PR\u0005\u0016\u0000\u0000QP\u0001\u0000\u0000\u0000" + + "RS\u0001\u0000\u0000\u0000SQ\u0001\u0000\u0000\u0000ST\u0001\u0000\u0000" + + "\u0000TW\u0001\u0000\u0000\u0000UW\u0005\u0015\u0000\u0000VI\u0001\u0000" + + "\u0000\u0000VM\u0001\u0000\u0000\u0000VQ\u0001\u0000\u0000\u0000VU\u0001" + + "\u0000\u0000\u0000W\u000b\u0001\u0000\u0000\u0000\n\u001d\'/1:FIMSV"; + + public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); + + protected static final DFA[] _decisionToDFA; + + protected static final PredictionContextCache _sharedContextCache = new PredictionContextCache(); + + private static final String[] _LITERAL_NAMES = makeLiteralNames(); + + private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); + + public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); + + static { + RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION); + } + + static { + tokenNames = new String[_SYMBOLIC_NAMES.length]; + for (int i = 0; i < tokenNames.length; i++) { + tokenNames[i] = VOCABULARY.getLiteralName(i); + if (tokenNames[i] == null) { + tokenNames[i] = VOCABULARY.getSymbolicName(i); + } + + if (tokenNames[i] == null) { + tokenNames[i] = ""; + } + } + } + + static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; + for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) { + _decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i); + } + } + + public FiltersParser(TokenStream input) { + super(input); + _interp = new ParserATNSimulator(this, _ATN, _decisionToDFA, _sharedContextCache); + } + + private static String[] makeRuleNames() { + return new String[] { "where", "booleanExpression", "constantArray", "compare", "identifier", "constant" }; + } + + private static String[] makeLiteralNames() { + return new String[] { null, null, "'.'", "','", "'['", "']'", "'('", "')'", "'=='", "'-'", "'+'", "'>'", "'>='", + "'<'", "'<='", "'!='" }; + } + + private static String[] makeSymbolicNames() { + return new String[] { null, "WHERE", "DOT", "COMMA", "LEFT_SQUARE_BRACKETS", "RIGHT_SQUARE_BRACKETS", + "LEFT_PARENTHESIS", "RIGHT_PARENTHESIS", "EQUALS", "MINUS", "PLUS", "GT", "GE", "LT", "LE", "NE", "AND", + "OR", "IN", "NIN", "NOT", "BOOLEAN_VALUE", "QUOTED_STRING", "INTEGER_VALUE", "DECIMAL_VALUE", + "IDENTIFIER", "WS" }; + } + + @Override + @Deprecated + public String[] getTokenNames() { + return tokenNames; + } + + @Override + + public Vocabulary getVocabulary() { + return VOCABULARY; + } + + @Override + public String getGrammarFileName() { + return "Filters.g4"; + } + + @Override + public String[] getRuleNames() { + return ruleNames; + } + + @Override + public String getSerializedATN() { + return _serializedATN; + } + + @Override + public ATN getATN() { + return _ATN; + } + + public final WhereContext where() throws RecognitionException { + WhereContext _localctx = new WhereContext(_ctx, getState()); + enterRule(_localctx, 0, RULE_where); + try { + enterOuterAlt(_localctx, 1); + { + setState(12); + match(WHERE); + setState(13); + booleanExpression(0); + setState(14); + match(EOF); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public final BooleanExpressionContext booleanExpression() throws RecognitionException { + return booleanExpression(0); + } + + private BooleanExpressionContext booleanExpression(int _p) throws RecognitionException { + ParserRuleContext _parentctx = _ctx; + int _parentState = getState(); + BooleanExpressionContext _localctx = new BooleanExpressionContext(_ctx, _parentState); + BooleanExpressionContext _prevctx = _localctx; + int _startState = 2; + enterRecursionRule(_localctx, 2, RULE_booleanExpression, _p); + try { + int _alt; + enterOuterAlt(_localctx, 1); + { + setState(39); + _errHandler.sync(this); + switch (getInterpreter().adaptivePredict(_input, 1, _ctx)) { + case 1: { + _localctx = new CompareExpressionContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + + setState(17); + identifier(); + setState(18); + compare(); + setState(19); + constant(); + } + break; + case 2: { + _localctx = new InExpressionContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(21); + identifier(); + setState(22); + match(IN); + setState(23); + constantArray(); + } + break; + case 3: { + _localctx = new NinExpressionContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(25); + identifier(); + setState(29); + _errHandler.sync(this); + switch (_input.LA(1)) { + case NOT: { + setState(26); + match(NOT); + setState(27); + match(IN); + } + break; + case NIN: { + setState(28); + match(NIN); + } + break; + default: + throw new NoViableAltException(this); + } + setState(31); + constantArray(); + } + break; + case 4: { + _localctx = new GroupExpressionContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(33); + match(LEFT_PARENTHESIS); + setState(34); + booleanExpression(0); + setState(35); + match(RIGHT_PARENTHESIS); + } + break; + case 5: { + _localctx = new NotExpressionContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(37); + match(NOT); + setState(38); + booleanExpression(1); + } + break; + } + _ctx.stop = _input.LT(-1); + setState(49); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input, 3, _ctx); + while (_alt != 2 && _alt != org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER) { + if (_alt == 1) { + if (_parseListeners != null) { + triggerExitRuleEvent(); + } + _prevctx = _localctx; + { + setState(47); + _errHandler.sync(this); + switch (getInterpreter().adaptivePredict(_input, 2, _ctx)) { + case 1: { + _localctx = new AndExpressionContext( + new BooleanExpressionContext(_parentctx, _parentState)); + ((AndExpressionContext) _localctx).left = _prevctx; + pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); + setState(41); + if (!(precpred(_ctx, 4))) { + throw new FailedPredicateException(this, "precpred(_ctx, 4)"); + } + setState(42); + ((AndExpressionContext) _localctx).operator = match(AND); + setState(43); + ((AndExpressionContext) _localctx).right = booleanExpression(5); + } + break; + case 2: { + _localctx = new OrExpressionContext( + new BooleanExpressionContext(_parentctx, _parentState)); + ((OrExpressionContext) _localctx).left = _prevctx; + pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); + setState(44); + if (!(precpred(_ctx, 3))) { + throw new FailedPredicateException(this, "precpred(_ctx, 3)"); + } + setState(45); + ((OrExpressionContext) _localctx).operator = match(OR); + setState(46); + ((OrExpressionContext) _localctx).right = booleanExpression(4); + } + break; + } + } + } + setState(51); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input, 3, _ctx); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + unrollRecursionContexts(_parentctx); + } + return _localctx; + } + + public final ConstantArrayContext constantArray() throws RecognitionException { + ConstantArrayContext _localctx = new ConstantArrayContext(_ctx, getState()); + enterRule(_localctx, 4, RULE_constantArray); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(52); + match(LEFT_SQUARE_BRACKETS); + setState(53); + constant(); + setState(58); + _errHandler.sync(this); + _la = _input.LA(1); + while (_la == COMMA) { + { + { + setState(54); + match(COMMA); + setState(55); + constant(); + } + } + setState(60); + _errHandler.sync(this); + _la = _input.LA(1); + } + setState(61); + match(RIGHT_SQUARE_BRACKETS); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public final CompareContext compare() throws RecognitionException { + CompareContext _localctx = new CompareContext(_ctx, getState()); + enterRule(_localctx, 6, RULE_compare); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(63); + _la = _input.LA(1); + if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & 63744L) != 0))) { + _errHandler.recoverInline(this); + } + else { + if (_input.LA(1) == Token.EOF) { + matchedEOF = true; + } + _errHandler.reportMatch(this); + consume(); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public final IdentifierContext identifier() throws RecognitionException { + IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); + enterRule(_localctx, 8, RULE_identifier); + try { + setState(70); + _errHandler.sync(this); + switch (getInterpreter().adaptivePredict(_input, 5, _ctx)) { + case 1: + enterOuterAlt(_localctx, 1); { + setState(65); + match(IDENTIFIER); + setState(66); + match(DOT); + setState(67); + match(IDENTIFIER); + } + break; + case 2: + enterOuterAlt(_localctx, 2); { + setState(68); + match(IDENTIFIER); + } + break; + case 3: + enterOuterAlt(_localctx, 3); { + setState(69); + match(QUOTED_STRING); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public final ConstantContext constant() throws RecognitionException { + ConstantContext _localctx = new ConstantContext(_ctx, getState()); + enterRule(_localctx, 10, RULE_constant); + int _la; + try { + int _alt; + setState(86); + _errHandler.sync(this); + switch (getInterpreter().adaptivePredict(_input, 9, _ctx)) { + case 1: + _localctx = new IntegerConstantContext(_localctx); + enterOuterAlt(_localctx, 1); { + setState(73); + _errHandler.sync(this); + _la = _input.LA(1); + if (_la == MINUS || _la == PLUS) { + { + setState(72); + _la = _input.LA(1); + if (!(_la == MINUS || _la == PLUS)) { + _errHandler.recoverInline(this); + } + else { + if (_input.LA(1) == Token.EOF) { + matchedEOF = true; + } + _errHandler.reportMatch(this); + consume(); + } + } + } + + setState(75); + match(INTEGER_VALUE); + } + break; + case 2: + _localctx = new DecimalConstantContext(_localctx); + enterOuterAlt(_localctx, 2); { + setState(77); + _errHandler.sync(this); + _la = _input.LA(1); + if (_la == MINUS || _la == PLUS) { + { + setState(76); + _la = _input.LA(1); + if (!(_la == MINUS || _la == PLUS)) { + _errHandler.recoverInline(this); + } + else { + if (_input.LA(1) == Token.EOF) { + matchedEOF = true; + } + _errHandler.reportMatch(this); + consume(); + } + } + } + + setState(79); + match(DECIMAL_VALUE); + } + break; + case 3: + _localctx = new TextConstantContext(_localctx); + enterOuterAlt(_localctx, 3); { + setState(81); + _errHandler.sync(this); + _alt = 1; + do { + switch (_alt) { + case 1: { + { + setState(80); + match(QUOTED_STRING); + } + } + break; + default: + throw new NoViableAltException(this); + } + setState(83); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input, 8, _ctx); + } + while (_alt != 2 && _alt != org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER); + } + break; + case 4: + _localctx = new BooleanConstantContext(_localctx); + enterOuterAlt(_localctx, 4); { + setState(85); + match(BOOLEAN_VALUE); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { + switch (ruleIndex) { + case 1: + return booleanExpression_sempred((BooleanExpressionContext) _localctx, predIndex); + } + return true; + } + + private boolean booleanExpression_sempred(BooleanExpressionContext _localctx, int predIndex) { + switch (predIndex) { + case 0: + return precpred(_ctx, 4); + case 1: + return precpred(_ctx, 3); + } + return true; + } + + @SuppressWarnings("CheckReturnValue") + public static class WhereContext extends ParserRuleContext { + + public WhereContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + + public TerminalNode WHERE() { + return getToken(FiltersParser.WHERE, 0); + } + + public BooleanExpressionContext booleanExpression() { + return getRuleContext(BooleanExpressionContext.class, 0); + } + + public TerminalNode EOF() { + return getToken(FiltersParser.EOF, 0); + } + + @Override + public int getRuleIndex() { + return RULE_where; + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterWhere(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitWhere(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitWhere(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class BooleanExpressionContext extends ParserRuleContext { + + public BooleanExpressionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + + public BooleanExpressionContext() { + } + + @Override + public int getRuleIndex() { + return RULE_booleanExpression; + } + + public void copyFrom(BooleanExpressionContext ctx) { + super.copyFrom(ctx); + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class NinExpressionContext extends BooleanExpressionContext { + + public NinExpressionContext(BooleanExpressionContext ctx) { + copyFrom(ctx); + } + + public IdentifierContext identifier() { + return getRuleContext(IdentifierContext.class, 0); + } + + public ConstantArrayContext constantArray() { + return getRuleContext(ConstantArrayContext.class, 0); + } + + public TerminalNode NOT() { + return getToken(FiltersParser.NOT, 0); + } + + public TerminalNode IN() { + return getToken(FiltersParser.IN, 0); + } + + public TerminalNode NIN() { + return getToken(FiltersParser.NIN, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterNinExpression(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitNinExpression(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitNinExpression(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class AndExpressionContext extends BooleanExpressionContext { + + public BooleanExpressionContext left; + + public Token operator; + + public BooleanExpressionContext right; + + public AndExpressionContext(BooleanExpressionContext ctx) { + copyFrom(ctx); + } + + public List booleanExpression() { + return getRuleContexts(BooleanExpressionContext.class); + } + + public BooleanExpressionContext booleanExpression(int i) { + return getRuleContext(BooleanExpressionContext.class, i); + } + + public TerminalNode AND() { + return getToken(FiltersParser.AND, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterAndExpression(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitAndExpression(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitAndExpression(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class InExpressionContext extends BooleanExpressionContext { + + public InExpressionContext(BooleanExpressionContext ctx) { + copyFrom(ctx); + } + + public IdentifierContext identifier() { + return getRuleContext(IdentifierContext.class, 0); + } + + public TerminalNode IN() { + return getToken(FiltersParser.IN, 0); + } + + public ConstantArrayContext constantArray() { + return getRuleContext(ConstantArrayContext.class, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterInExpression(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitInExpression(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitInExpression(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class NotExpressionContext extends BooleanExpressionContext { + + public NotExpressionContext(BooleanExpressionContext ctx) { + copyFrom(ctx); + } + + public TerminalNode NOT() { + return getToken(FiltersParser.NOT, 0); + } + + public BooleanExpressionContext booleanExpression() { + return getRuleContext(BooleanExpressionContext.class, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterNotExpression(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitNotExpression(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitNotExpression(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class CompareExpressionContext extends BooleanExpressionContext { + + public CompareExpressionContext(BooleanExpressionContext ctx) { + copyFrom(ctx); + } + + public IdentifierContext identifier() { + return getRuleContext(IdentifierContext.class, 0); + } + + public CompareContext compare() { + return getRuleContext(CompareContext.class, 0); + } + + public ConstantContext constant() { + return getRuleContext(ConstantContext.class, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterCompareExpression(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitCompareExpression(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitCompareExpression(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class OrExpressionContext extends BooleanExpressionContext { + + public BooleanExpressionContext left; + + public Token operator; + + public BooleanExpressionContext right; + + public OrExpressionContext(BooleanExpressionContext ctx) { + copyFrom(ctx); + } + + public List booleanExpression() { + return getRuleContexts(BooleanExpressionContext.class); + } + + public BooleanExpressionContext booleanExpression(int i) { + return getRuleContext(BooleanExpressionContext.class, i); + } + + public TerminalNode OR() { + return getToken(FiltersParser.OR, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterOrExpression(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitOrExpression(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitOrExpression(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class GroupExpressionContext extends BooleanExpressionContext { + + public GroupExpressionContext(BooleanExpressionContext ctx) { + copyFrom(ctx); + } + + public TerminalNode LEFT_PARENTHESIS() { + return getToken(FiltersParser.LEFT_PARENTHESIS, 0); + } + + public BooleanExpressionContext booleanExpression() { + return getRuleContext(BooleanExpressionContext.class, 0); + } + + public TerminalNode RIGHT_PARENTHESIS() { + return getToken(FiltersParser.RIGHT_PARENTHESIS, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterGroupExpression(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitGroupExpression(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitGroupExpression(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class ConstantArrayContext extends ParserRuleContext { + + public ConstantArrayContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + + public TerminalNode LEFT_SQUARE_BRACKETS() { + return getToken(FiltersParser.LEFT_SQUARE_BRACKETS, 0); + } + + public List constant() { + return getRuleContexts(ConstantContext.class); + } + + public ConstantContext constant(int i) { + return getRuleContext(ConstantContext.class, i); + } + + public TerminalNode RIGHT_SQUARE_BRACKETS() { + return getToken(FiltersParser.RIGHT_SQUARE_BRACKETS, 0); + } + + public List COMMA() { + return getTokens(FiltersParser.COMMA); + } + + public TerminalNode COMMA(int i) { + return getToken(FiltersParser.COMMA, i); + } + + @Override + public int getRuleIndex() { + return RULE_constantArray; + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterConstantArray(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitConstantArray(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitConstantArray(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class CompareContext extends ParserRuleContext { + + public CompareContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + + public TerminalNode EQUALS() { + return getToken(FiltersParser.EQUALS, 0); + } + + public TerminalNode GT() { + return getToken(FiltersParser.GT, 0); + } + + public TerminalNode GE() { + return getToken(FiltersParser.GE, 0); + } + + public TerminalNode LT() { + return getToken(FiltersParser.LT, 0); + } + + public TerminalNode LE() { + return getToken(FiltersParser.LE, 0); + } + + public TerminalNode NE() { + return getToken(FiltersParser.NE, 0); + } + + @Override + public int getRuleIndex() { + return RULE_compare; + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterCompare(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitCompare(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitCompare(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class IdentifierContext extends ParserRuleContext { + + public IdentifierContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + + public List IDENTIFIER() { + return getTokens(FiltersParser.IDENTIFIER); + } + + public TerminalNode IDENTIFIER(int i) { + return getToken(FiltersParser.IDENTIFIER, i); + } + + public TerminalNode DOT() { + return getToken(FiltersParser.DOT, 0); + } + + public TerminalNode QUOTED_STRING() { + return getToken(FiltersParser.QUOTED_STRING, 0); + } + + @Override + public int getRuleIndex() { + return RULE_identifier; + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterIdentifier(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitIdentifier(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitIdentifier(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class ConstantContext extends ParserRuleContext { + + public ConstantContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + + public ConstantContext() { + } + + @Override + public int getRuleIndex() { + return RULE_constant; + } + + public void copyFrom(ConstantContext ctx) { + super.copyFrom(ctx); + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class DecimalConstantContext extends ConstantContext { + + public DecimalConstantContext(ConstantContext ctx) { + copyFrom(ctx); + } + + public TerminalNode DECIMAL_VALUE() { + return getToken(FiltersParser.DECIMAL_VALUE, 0); + } + + public TerminalNode MINUS() { + return getToken(FiltersParser.MINUS, 0); + } + + public TerminalNode PLUS() { + return getToken(FiltersParser.PLUS, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterDecimalConstant(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitDecimalConstant(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitDecimalConstant(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class TextConstantContext extends ConstantContext { + + public TextConstantContext(ConstantContext ctx) { + copyFrom(ctx); + } + + public List QUOTED_STRING() { + return getTokens(FiltersParser.QUOTED_STRING); + } + + public TerminalNode QUOTED_STRING(int i) { + return getToken(FiltersParser.QUOTED_STRING, i); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterTextConstant(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitTextConstant(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitTextConstant(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class BooleanConstantContext extends ConstantContext { + + public BooleanConstantContext(ConstantContext ctx) { + copyFrom(ctx); + } + + public TerminalNode BOOLEAN_VALUE() { + return getToken(FiltersParser.BOOLEAN_VALUE, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterBooleanConstant(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitBooleanConstant(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitBooleanConstant(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + + @SuppressWarnings("CheckReturnValue") + public static class IntegerConstantContext extends ConstantContext { + + public IntegerConstantContext(ConstantContext ctx) { + copyFrom(ctx); + } + + public TerminalNode INTEGER_VALUE() { + return getToken(FiltersParser.INTEGER_VALUE, 0); + } + + public TerminalNode MINUS() { + return getToken(FiltersParser.MINUS, 0); + } + + public TerminalNode PLUS() { + return getToken(FiltersParser.PLUS, 0); + } + + @Override + public void enterRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).enterIntegerConstant(this); + } + } + + @Override + public void exitRule(ParseTreeListener listener) { + if (listener instanceof FiltersListener) { + ((FiltersListener) listener).exitIntegerConstant(this); + } + } + + @Override + public T accept(ParseTreeVisitor visitor) { + if (visitor instanceof FiltersVisitor) { + return ((FiltersVisitor) visitor).visitIntegerConstant(this); + } + else { + return visitor.visitChildren(this); + } + } + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersVisitor.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersVisitor.java new file mode 100644 index 000000000..887159c2b --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/antlr4/FiltersVisitor.java @@ -0,0 +1,152 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.antlr4; + +// Generated from org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 by ANTLR 4.13.1 + +import org.antlr.v4.runtime.tree.ParseTreeVisitor; + +// ############################################################ +// # NOTE: This is ANTLR4 auto-generated code. Do not modify! # +// ############################################################ + +/** + * This interface defines a complete generic visitor for a parse tree produced by + * {@link FiltersParser}. + * + * @param The return type of the visit operation. Use {@link Void} for operations with + * no return type. + */ +public interface FiltersVisitor extends ParseTreeVisitor { + + /** + * Visit a parse tree produced by {@link FiltersParser#where}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitWhere(FiltersParser.WhereContext ctx); + + /** + * Visit a parse tree produced by the {@code NinExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitNinExpression(FiltersParser.NinExpressionContext ctx); + + /** + * Visit a parse tree produced by the {@code AndExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitAndExpression(FiltersParser.AndExpressionContext ctx); + + /** + * Visit a parse tree produced by the {@code InExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitInExpression(FiltersParser.InExpressionContext ctx); + + /** + * Visit a parse tree produced by the {@code NotExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitNotExpression(FiltersParser.NotExpressionContext ctx); + + /** + * Visit a parse tree produced by the {@code CompareExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitCompareExpression(FiltersParser.CompareExpressionContext ctx); + + /** + * Visit a parse tree produced by the {@code OrExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitOrExpression(FiltersParser.OrExpressionContext ctx); + + /** + * Visit a parse tree produced by the {@code GroupExpression} labeled alternative in + * {@link FiltersParser#booleanExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitGroupExpression(FiltersParser.GroupExpressionContext ctx); + + /** + * Visit a parse tree produced by {@link FiltersParser#constantArray}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitConstantArray(FiltersParser.ConstantArrayContext ctx); + + /** + * Visit a parse tree produced by {@link FiltersParser#compare}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitCompare(FiltersParser.CompareContext ctx); + + /** + * Visit a parse tree produced by {@link FiltersParser#identifier}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitIdentifier(FiltersParser.IdentifierContext ctx); + + /** + * Visit a parse tree produced by the {@code IntegerConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitIntegerConstant(FiltersParser.IntegerConstantContext ctx); + + /** + * Visit a parse tree produced by the {@code DecimalConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitDecimalConstant(FiltersParser.DecimalConstantContext ctx); + + /** + * Visit a parse tree produced by the {@code TextConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitTextConstant(FiltersParser.TextConstantContext ctx); + + /** + * Visit a parse tree produced by the {@code BooleanConstant} labeled alternative in + * {@link FiltersParser#constant}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitBooleanConstant(FiltersParser.BooleanConstantContext ctx); + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/AbstractFilterExpressionConverter.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/AbstractFilterExpressionConverter.java new file mode 100644 index 000000000..3d63d1217 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/AbstractFilterExpressionConverter.java @@ -0,0 +1,229 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.converter; + +import java.util.List; + +import org.springframework.ai.vectorstore.filter.Filter; +import org.springframework.ai.vectorstore.filter.Filter.Expression; +import org.springframework.ai.vectorstore.filter.Filter.ExpressionType; +import org.springframework.ai.vectorstore.filter.Filter.Group; +import org.springframework.ai.vectorstore.filter.Filter.Operand; +import org.springframework.ai.vectorstore.filter.FilterExpressionConverter; +import org.springframework.ai.vectorstore.filter.FilterHelper; + +/** + * AbstractFilterExpressionConverter is an abstract class that implements the + * FilterExpressionConverter interface. It provides default implementations for converting + * a Filter.Expression into a string representation. All specific filter expression + * converters should extend this abstract class and implement the remaining abstract + * methods. Note: The class cannot be directly instantiated as it is abstract. + * + * @author Christian Tzolov + */ +public abstract class AbstractFilterExpressionConverter implements FilterExpressionConverter { + + /** + * Create a new AbstractFilterExpressionConverter. + */ + public AbstractFilterExpressionConverter() { + } + + @Override + public String convertExpression(Expression expression) { + return this.convertOperand(expression); + } + + /** + * Convert the given operand into a string representation. + * @param operand the operand to convert + * @return the string representation of the operand + */ + protected String convertOperand(Operand operand) { + var context = new StringBuilder(); + this.convertOperand(operand, context); + return context.toString(); + } + + /** + * Convert the given operand into a string representation. + * @param operand the operand to convert + * @param context the context to append the string representation to + */ + protected void convertOperand(Operand operand, StringBuilder context) { + + if (operand instanceof Filter.Group group) { + this.doGroup(group, context); + } + else if (operand instanceof Filter.Key key) { + this.doKey(key, context); + } + else if (operand instanceof Filter.Value value) { + this.doValue(value, context); + } + else if (operand instanceof Filter.Expression expression) { + if ((expression.type() != ExpressionType.NOT && expression.type() != ExpressionType.AND + && expression.type() != ExpressionType.OR) && !(expression.right() instanceof Filter.Value)) { + throw new RuntimeException("Non AND/OR expression must have Value right argument!"); + } + if (expression.type() == ExpressionType.NOT) { + this.doNot(expression, context); + } + else { + this.doExpression(expression, context); + } + } + } + + /** + * Convert the given expression into a string representation. + * @param expression the expression to convert + * @param context the context to append the string representation to + */ + protected void doNot(Filter.Expression expression, StringBuilder context) { + // Default behavior is to convert the NOT expression into its semantically + // equivalent negation expression. + // Effectively removing the NOT types form the boolean expression tree before + // passing it to the doExpression. + this.convertOperand(FilterHelper.negate(expression), context); + } + + /** + * Convert the given expression into a string representation. + * @param expression the expression to convert + * @param context the context to append the string representation to + */ + protected abstract void doExpression(Filter.Expression expression, StringBuilder context); + + /** + * Convert the given key into a string representation. + * @param filterKey the key to convert + * @param context the context to append the string representation to + */ + protected abstract void doKey(Filter.Key filterKey, StringBuilder context); + + /** + * Convert the given value into a string representation. + * @param filterValue the value to convert + * @param context the context to append the string representation to + */ + protected void doValue(Filter.Value filterValue, StringBuilder context) { + if (filterValue.value() instanceof List list) { + doStartValueRange(filterValue, context); + int c = 0; + for (Object v : list) { + this.doSingleValue(v, context); + if (c++ < list.size() - 1) { + this.doAddValueRangeSpitter(filterValue, context); + } + } + this.doEndValueRange(filterValue, context); + } + else { + this.doSingleValue(filterValue.value(), context); + } + } + + /** + * Convert the given value into a string representation. + * @param value the value to convert + * @param context the context to append the string representation to + */ + protected void doSingleValue(Object value, StringBuilder context) { + if (value instanceof String) { + context.append(String.format("\"%s\"", value)); + } + else { + context.append(value); + } + } + + /** + * Convert the given group into a string representation. + * @param group the group to convert + * @param context the context to append the string representation to + */ + protected void doGroup(Group group, StringBuilder context) { + this.doStartGroup(group, context); + this.convertOperand(group.content(), context); + this.doEndGroup(group, context); + } + + /** + * Convert the given group into a string representation. + * @param group the group to convert + * @param context the context to append the string representation to + */ + protected void doStartGroup(Group group, StringBuilder context) { + } + + /** + * Convert the given group into a string representation. + * @param group the group to convert + * @param context the context to append the string representation to + */ + protected void doEndGroup(Group group, StringBuilder context) { + } + + /** + * Convert the given value range into a string representation. + * @param listValue the value range to convert + * @param context the context to append the string representation to + */ + protected void doStartValueRange(Filter.Value listValue, StringBuilder context) { + context.append("["); + } + + /** + * Convert the given value range into a string representation. + * @param listValue the value range to convert + * @param context the context to append the string representation to + */ + protected void doEndValueRange(Filter.Value listValue, StringBuilder context) { + context.append("]"); + } + + /** + * Convert the given value range into a string representation. + * @param listValue the value range to convert + * @param context the context to append the string representation to + */ + protected void doAddValueRangeSpitter(Filter.Value listValue, StringBuilder context) { + context.append(","); + } + + // Utilities + /** + * Check if the given string has outer quotes. + * @param str the string to check + * @return true if the string has outer quotes, false otherwise + */ + protected boolean hasOuterQuotes(String str) { + str = str.trim(); + return (str.startsWith("\"") && str.endsWith("\"")) || (str.startsWith("'") && str.endsWith("'")); + } + + /** + * Remove the outer quotes from the given string. + * @param in the string to remove the outer quotes from + * @return the string without the outer quotes + */ + protected String removeOuterQuotes(String in) { + return in.substring(1, in.length() - 1); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverter.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverter.java new file mode 100644 index 000000000..64877fc24 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/PineconeFilterExpressionConverter.java @@ -0,0 +1,64 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.converter; + +import org.springframework.ai.vectorstore.filter.Filter.Expression; +import org.springframework.ai.vectorstore.filter.Filter.ExpressionType; +import org.springframework.ai.vectorstore.filter.Filter.Key; + +/** + * Converts {@link Expression} into Pinecone metadata filter expression format. + * (https://docs.pinecone.io/docs/metadata-filtering) + * + * @author Christian Tzolov + */ +public class PineconeFilterExpressionConverter extends AbstractFilterExpressionConverter { + + @Override + protected void doExpression(Expression exp, StringBuilder context) { + + context.append("{"); + if (exp.type() == ExpressionType.AND || exp.type() == ExpressionType.OR) { + context.append(getOperationSymbol(exp)); + context.append("["); + this.convertOperand(exp.left(), context); + context.append(","); + this.convertOperand(exp.right(), context); + context.append("]"); + } + else { + this.convertOperand(exp.left(), context); + context.append("{"); + context.append(getOperationSymbol(exp)); + this.convertOperand(exp.right(), context); + context.append("}"); + } + context.append("}"); + + } + + private String getOperationSymbol(Expression exp) { + return "\"$" + exp.type().toString().toLowerCase() + "\": "; + } + + @Override + protected void doKey(Key key, StringBuilder context) { + var identifier = (hasOuterQuotes(key.key())) ? removeOuterQuotes(key.key()) : key.key(); + context.append("\"" + identifier + "\": "); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/PrintFilterExpressionConverter.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/PrintFilterExpressionConverter.java new file mode 100644 index 000000000..14d2d1216 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/converter/PrintFilterExpressionConverter.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.converter; + +import org.springframework.ai.vectorstore.filter.Filter.Expression; +import org.springframework.ai.vectorstore.filter.Filter.Group; +import org.springframework.ai.vectorstore.filter.Filter.Key; + +/** + * Converts {@link Expression} into test string format. + * + * @author Christian Tzolov + */ +public class PrintFilterExpressionConverter extends AbstractFilterExpressionConverter { + + public void doExpression(Expression expression, StringBuilder context) { + this.convertOperand(expression.left(), context); + context.append(" " + expression.type() + " "); + this.convertOperand(expression.right(), context); + + } + + public void doKey(Key key, StringBuilder context) { + context.append(key.key()); + } + + @Override + public void doStartGroup(Group group, StringBuilder context) { + context.append("("); + } + + @Override + public void doEndGroup(Group group, StringBuilder context) { + context.append(")"); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/AbstractObservationVectorStore.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/AbstractObservationVectorStore.java new file mode 100644 index 000000000..17a15a5e5 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/AbstractObservationVectorStore.java @@ -0,0 +1,154 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import java.util.List; +import java.util.Optional; + +import io.micrometer.observation.ObservationRegistry; + +import org.springframework.ai.document.Document; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.lang.Nullable; + +/** + * Abstract base class for {@link VectorStore} implementations that provides observation + * capabilities. + * + * @author Christian Tzolov + * @author Soby Chacko + * @since 1.0.0 + */ +public abstract class AbstractObservationVectorStore implements VectorStore { + + private static final VectorStoreObservationConvention DEFAULT_OBSERVATION_CONVENTION = new DefaultVectorStoreObservationConvention(); + + private final ObservationRegistry observationRegistry; + + @Nullable + private final VectorStoreObservationConvention customObservationConvention; + + @Nullable + protected final EmbeddingModel embeddingModel; + + /** + * Create a new {@link AbstractObservationVectorStore} instance. + * @param observationRegistry the observation registry to use + * @param customObservationConvention the custom observation convention to use + */ + @Deprecated(since = "1.0.0-M5", forRemoval = true) + public AbstractObservationVectorStore(ObservationRegistry observationRegistry, + @Nullable VectorStoreObservationConvention customObservationConvention) { + this(null, observationRegistry, customObservationConvention); + } + + private AbstractObservationVectorStore(@Nullable EmbeddingModel embeddingModel, + ObservationRegistry observationRegistry, + @Nullable VectorStoreObservationConvention customObservationConvention) { + this.embeddingModel = embeddingModel; + this.observationRegistry = observationRegistry; + this.customObservationConvention = customObservationConvention; + } + + /** + * Creates a new AbstractObservationVectorStore instance with the specified builder + * settings. Initializes observation-related components and the embedding model. + * @param builder the builder containing configuration settings + */ + public AbstractObservationVectorStore(AbstractVectorStoreBuilder builder) { + this(builder.getEmbeddingModel(), builder.getObservationRegistry(), builder.getCustomObservationConvention()); + } + + /** + * Create a new {@link AbstractObservationVectorStore} instance. + * @param documents the documents to add + */ + @Override + public void add(List documents) { + + VectorStoreObservationContext observationContext = this + .createObservationContextBuilder(VectorStoreObservationContext.Operation.ADD.value()) + .build(); + + VectorStoreObservationDocumentation.AI_VECTOR_STORE + .observation(this.customObservationConvention, DEFAULT_OBSERVATION_CONVENTION, () -> observationContext, + this.observationRegistry) + .observe(() -> this.doAdd(documents)); + } + + @Override + public Optional delete(List deleteDocIds) { + + VectorStoreObservationContext observationContext = this + .createObservationContextBuilder(VectorStoreObservationContext.Operation.DELETE.value()) + .build(); + + return VectorStoreObservationDocumentation.AI_VECTOR_STORE + .observation(this.customObservationConvention, DEFAULT_OBSERVATION_CONVENTION, () -> observationContext, + this.observationRegistry) + .observe(() -> this.doDelete(deleteDocIds)); + } + + @Override + public List similaritySearch(SearchRequest request) { + + VectorStoreObservationContext searchObservationContext = this + .createObservationContextBuilder(VectorStoreObservationContext.Operation.QUERY.value()) + .withQueryRequest(request) + .build(); + + return VectorStoreObservationDocumentation.AI_VECTOR_STORE + .observation(this.customObservationConvention, DEFAULT_OBSERVATION_CONVENTION, + () -> searchObservationContext, this.observationRegistry) + .observe(() -> { + var documents = this.doSimilaritySearch(request); + searchObservationContext.setQueryResponse(documents); + return documents; + }); + } + + /** + * Perform the actual add operation. + * @param documents the documents to add + */ + public abstract void doAdd(List documents); + + /** + * Perform the actual delete operation. + * @param idList the list of document IDs to delete + * @return true if the documents were successfully deleted + */ + public abstract Optional doDelete(List idList); + + /** + * Perform the actual similarity search operation. + * @param request the search request + * @return the list of documents that match the query request conditions + */ + public abstract List doSimilaritySearch(SearchRequest request); + + /** + * Create a new {@link VectorStoreObservationContext.Builder} instance. + * @param operationName the operation name + * @return the observation context builder + */ + public abstract VectorStoreObservationContext.Builder createObservationContextBuilder(String operationName); + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConvention.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConvention.java new file mode 100644 index 000000000..15700d357 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConvention.java @@ -0,0 +1,161 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import io.micrometer.common.KeyValue; +import io.micrometer.common.KeyValues; + +import org.springframework.ai.observation.conventions.SpringAiKind; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.HighCardinalityKeyNames; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.LowCardinalityKeyNames; +import org.springframework.lang.Nullable; +import org.springframework.util.StringUtils; + +/** + * Default conventions to populate observations for vector store operations. + * + * @author Christian Tzolov + * @author Thomas Vitale + * @since 1.0.0 + */ +public class DefaultVectorStoreObservationConvention implements VectorStoreObservationConvention { + + public static final String DEFAULT_NAME = "db.vector.client.operation"; + + private final String name; + + public DefaultVectorStoreObservationConvention() { + this(DEFAULT_NAME); + } + + public DefaultVectorStoreObservationConvention(String name) { + this.name = name; + } + + @Override + public String getName() { + return this.name; + } + + @Override + @Nullable + public String getContextualName(VectorStoreObservationContext context) { + return "%s %s".formatted(context.getDatabaseSystem(), context.getOperationName()); + } + + @Override + public KeyValues getLowCardinalityKeyValues(VectorStoreObservationContext context) { + return KeyValues.of(springAiKind(), dbSystem(context), dbOperationName(context)); + } + + protected KeyValue springAiKind() { + return KeyValue.of(LowCardinalityKeyNames.SPRING_AI_KIND, SpringAiKind.VECTOR_STORE.value()); + } + + protected KeyValue dbSystem(VectorStoreObservationContext context) { + return KeyValue.of(LowCardinalityKeyNames.DB_SYSTEM, context.getDatabaseSystem()); + } + + protected KeyValue dbOperationName(VectorStoreObservationContext context) { + return KeyValue.of(LowCardinalityKeyNames.DB_OPERATION_NAME, context.getOperationName()); + } + + @Override + public KeyValues getHighCardinalityKeyValues(VectorStoreObservationContext context) { + var keyValues = KeyValues.empty(); + keyValues = collectionName(keyValues, context); + keyValues = dimensions(keyValues, context); + keyValues = fieldName(keyValues, context); + keyValues = metadataFilter(keyValues, context); + keyValues = namespace(keyValues, context); + keyValues = queryContent(keyValues, context); + keyValues = similarityMetric(keyValues, context); + keyValues = similarityThreshold(keyValues, context); + keyValues = topK(keyValues, context); + return keyValues; + } + + protected KeyValues collectionName(KeyValues keyValues, VectorStoreObservationContext context) { + if (StringUtils.hasText(context.getCollectionName())) { + return keyValues.and(HighCardinalityKeyNames.DB_COLLECTION_NAME.asString(), context.getCollectionName()); + } + return keyValues; + } + + protected KeyValues dimensions(KeyValues keyValues, VectorStoreObservationContext context) { + if (context.getDimensions() != null && context.getDimensions() > 0) { + return keyValues.and(HighCardinalityKeyNames.DB_VECTOR_DIMENSION_COUNT.asString(), + "" + context.getDimensions()); + } + return keyValues; + } + + protected KeyValues fieldName(KeyValues keyValues, VectorStoreObservationContext context) { + if (StringUtils.hasText(context.getFieldName())) { + return keyValues.and(HighCardinalityKeyNames.DB_VECTOR_FIELD_NAME.asString(), context.getFieldName()); + } + return keyValues; + } + + protected KeyValues metadataFilter(KeyValues keyValues, VectorStoreObservationContext context) { + if (context.getQueryRequest() != null && context.getQueryRequest().getFilterExpression() != null) { + return keyValues.and(HighCardinalityKeyNames.DB_VECTOR_QUERY_FILTER.asString(), + context.getQueryRequest().getFilterExpression().toString()); + } + return keyValues; + } + + protected KeyValues namespace(KeyValues keyValues, VectorStoreObservationContext context) { + if (StringUtils.hasText(context.getNamespace())) { + return keyValues.and(HighCardinalityKeyNames.DB_NAMESPACE.asString(), context.getNamespace()); + } + return keyValues; + } + + protected KeyValues queryContent(KeyValues keyValues, VectorStoreObservationContext context) { + if (context.getQueryRequest() != null && StringUtils.hasText(context.getQueryRequest().getQuery())) { + return keyValues.and(HighCardinalityKeyNames.DB_VECTOR_QUERY_CONTENT.asString(), + context.getQueryRequest().getQuery()); + } + return keyValues; + } + + protected KeyValues similarityMetric(KeyValues keyValues, VectorStoreObservationContext context) { + if (StringUtils.hasText(context.getSimilarityMetric())) { + return keyValues.and(HighCardinalityKeyNames.DB_SEARCH_SIMILARITY_METRIC.asString(), + context.getSimilarityMetric()); + } + return keyValues; + } + + protected KeyValues similarityThreshold(KeyValues keyValues, VectorStoreObservationContext context) { + if (context.getQueryRequest() != null && context.getQueryRequest().getSimilarityThreshold() >= 0) { + return keyValues.and(HighCardinalityKeyNames.DB_VECTOR_QUERY_SIMILARITY_THRESHOLD.asString(), + String.valueOf(context.getQueryRequest().getSimilarityThreshold())); + } + return keyValues; + } + + protected KeyValues topK(KeyValues keyValues, VectorStoreObservationContext context) { + if (context.getQueryRequest() != null && context.getQueryRequest().getTopK() > 0) { + return keyValues.and(HighCardinalityKeyNames.DB_VECTOR_QUERY_TOP_K.asString(), + "" + context.getQueryRequest().getTopK()); + } + return keyValues; + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContentProcessor.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContentProcessor.java new file mode 100644 index 000000000..e73ca4603 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContentProcessor.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import java.util.List; + +import org.springframework.ai.document.Document; +import org.springframework.util.CollectionUtils; + +/** + * Utilities to process the query content in observations for vector store operations. + * + * @author Thomas Vitale + */ +public final class VectorStoreObservationContentProcessor { + + private VectorStoreObservationContentProcessor() { + } + + public static List documents(VectorStoreObservationContext context) { + if (CollectionUtils.isEmpty(context.getQueryResponse())) { + return List.of(); + } + + return context.getQueryResponse().stream().map(Document::getText).toList(); + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContext.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContext.java new file mode 100644 index 000000000..832d3355f --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContext.java @@ -0,0 +1,228 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import java.util.List; + +import io.micrometer.observation.Observation; + +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Context used to store metadata for vector store operations. + * + * @author Christian Tzolov + * @author Thomas Vitale + * @since 1.0.0 + */ +public class VectorStoreObservationContext extends Observation.Context { + + private final String databaseSystem; + + // COMMON + + private final String operationName; + + @Nullable + private String collectionName; + + @Nullable + private Integer dimensions; + + @Nullable + private String fieldName; + + @Nullable + private String namespace; + + @Nullable + private String similarityMetric; + + @Nullable + private SearchRequest queryRequest; + + // SEARCH + + @Nullable + private List queryResponse; + + public VectorStoreObservationContext(String databaseSystem, String operationName) { + Assert.hasText(databaseSystem, "databaseSystem cannot be null or empty"); + Assert.hasText(operationName, "operationName cannot be null or empty"); + this.databaseSystem = databaseSystem; + this.operationName = operationName; + } + + public static Builder builder(String databaseSystem, String operationName) { + return new Builder(databaseSystem, operationName); + } + + public static Builder builder(String databaseSystem, Operation operation) { + return builder(databaseSystem, operation.value); + } + + public String getDatabaseSystem() { + return this.databaseSystem; + } + + public String getOperationName() { + return this.operationName; + } + + @Nullable + public String getCollectionName() { + return this.collectionName; + } + + public void setCollectionName(@Nullable String collectionName) { + this.collectionName = collectionName; + } + + @Nullable + public Integer getDimensions() { + return this.dimensions; + } + + public void setDimensions(@Nullable Integer dimensions) { + this.dimensions = dimensions; + } + + @Nullable + public String getFieldName() { + return this.fieldName; + } + + public void setFieldName(@Nullable String fieldName) { + this.fieldName = fieldName; + } + + @Nullable + public String getNamespace() { + return this.namespace; + } + + public void setNamespace(@Nullable String namespace) { + this.namespace = namespace; + } + + @Nullable + public String getSimilarityMetric() { + return this.similarityMetric; + } + + public void setSimilarityMetric(@Nullable String similarityMetric) { + this.similarityMetric = similarityMetric; + } + + @Nullable + public SearchRequest getQueryRequest() { + return this.queryRequest; + } + + public void setQueryRequest(@Nullable SearchRequest queryRequest) { + this.queryRequest = queryRequest; + } + + @Nullable + public List getQueryResponse() { + return this.queryResponse; + } + + public void setQueryResponse(@Nullable List queryResponse) { + this.queryResponse = queryResponse; + } + + public enum Operation { + + /** + * VectorStore add operation. + */ + ADD("add"), + /** + * VectorStore delete operation. + */ + DELETE("delete"), + /** + * VectorStore similarity search operation. + */ + QUERY("query"); + + public final String value; + + Operation(String value) { + this.value = value; + } + + public String value() { + return this.value; + } + + } + + public static class Builder { + + private final VectorStoreObservationContext context; + + public Builder(String databaseSystem, String operationName) { + this.context = new VectorStoreObservationContext(databaseSystem, operationName); + } + + public Builder withCollectionName(String collectionName) { + this.context.setCollectionName(collectionName); + return this; + } + + public Builder withDimensions(Integer dimensions) { + this.context.setDimensions(dimensions); + return this; + } + + public Builder withFieldName(@Nullable String fieldName) { + this.context.setFieldName(fieldName); + return this; + } + + public Builder withNamespace(String namespace) { + this.context.setNamespace(namespace); + return this; + } + + public Builder withQueryRequest(SearchRequest request) { + this.context.setQueryRequest(request); + return this; + } + + public Builder withQueryResponse(List documents) { + this.context.setQueryResponse(documents); + return this; + } + + public Builder withSimilarityMetric(String similarityMetric) { + this.context.setSimilarityMetric(similarityMetric); + return this; + } + + public VectorStoreObservationContext build() { + return this.context; + } + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationConvention.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationConvention.java new file mode 100644 index 000000000..ed2afd5f3 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationConvention.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationConvention; + +/** + * A {@link ObservationConvention} for {@link VectorStoreObservationContext}. + * + * @author Christian Tzolov + * @since 1.0.0 + */ + +public interface VectorStoreObservationConvention extends ObservationConvention { + + @Override + default boolean supportsContext(Observation.Context context) { + return context instanceof VectorStoreObservationContext; + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationDocumentation.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationDocumentation.java new file mode 100644 index 000000000..f351ca292 --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationDocumentation.java @@ -0,0 +1,208 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import io.micrometer.common.docs.KeyName; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationConvention; +import io.micrometer.observation.docs.ObservationDocumentation; + +import org.springframework.ai.observation.conventions.VectorStoreObservationAttributes; + +/** + * Documented conventions for vector store observations. + * + * @author Christian Tzolov + * @author Thomas Vitale + * @since 1.0.0 + */ +public enum VectorStoreObservationDocumentation implements ObservationDocumentation { + + /** + * Vector Store observations for clients. + */ + AI_VECTOR_STORE { + @Override + public Class> getDefaultConvention() { + return DefaultVectorStoreObservationConvention.class; + } + + @Override + public KeyName[] getLowCardinalityKeyNames() { + return LowCardinalityKeyNames.values(); + } + + @Override + public KeyName[] getHighCardinalityKeyNames() { + return HighCardinalityKeyNames.values(); + } + }; + + /** + * Low-cardinality observation key names for vector store operations. + */ + public enum LowCardinalityKeyNames implements KeyName { + + /** + * Spring AI kind. + */ + SPRING_AI_KIND { + @Override + public String asString() { + return "spring.ai.kind"; + } + }, + + /** + * The name of the operation or command being executed. + */ + DB_OPERATION_NAME { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_OPERATION_NAME.value(); + } + }, + + /** + * The database management system (DBMS) product as identified by the client + * instrumentation. + */ + DB_SYSTEM { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_SYSTEM.value(); + } + } + + } + + /** + * High-cardinality observation key names for vector store operations. + */ + public enum HighCardinalityKeyNames implements KeyName { + + // DB General + + /** + * The name of a collection (table, container) within the database. + */ + DB_COLLECTION_NAME { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_COLLECTION_NAME.value(); + } + }, + + /** + * The namespace of the database. + */ + DB_NAMESPACE { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_NAMESPACE.value(); + } + }, + + // DB Search + + /** + * The metric used in similarity search. + */ + DB_SEARCH_SIMILARITY_METRIC { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_SEARCH_SIMILARITY_METRIC.value(); + } + }, + + // DB Vector + + /** + * The dimension of the vector. + */ + DB_VECTOR_DIMENSION_COUNT { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_VECTOR_DIMENSION_COUNT.value(); + } + }, + + /** + * The name field as of the vector (e.g. a field name). + */ + DB_VECTOR_FIELD_NAME { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_VECTOR_FIELD_NAME.value(); + } + }, + + /** + * The content of the search query being executed. + */ + DB_VECTOR_QUERY_CONTENT { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_VECTOR_QUERY_CONTENT.value(); + } + }, + + /** + * The metadata filters used in the search query. + */ + DB_VECTOR_QUERY_FILTER { + @Override + public String asString() { + return "db.vector.query.filter"; + } + }, + + /** + * Returned documents from a similarity search query. + */ + DB_VECTOR_QUERY_RESPONSE_DOCUMENTS { + @Override + public String asString() { + return "db.vector.query.response.documents"; + } + }, + + /** + * Similarity threshold that accepts all search scores. A threshold value of 0.0 + * means any similarity is accepted or disable the similarity threshold filtering. + * A threshold value of 1.0 means an exact match is required. + */ + DB_VECTOR_QUERY_SIMILARITY_THRESHOLD { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_VECTOR_QUERY_SIMILARITY_THRESHOLD.value(); + } + }, + + /** + * The top-k most similar vectors returned by a query. + */ + DB_VECTOR_QUERY_TOP_K { + @Override + public String asString() { + return VectorStoreObservationAttributes.DB_VECTOR_QUERY_TOP_K.value(); + } + } + + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilter.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilter.java new file mode 100644 index 000000000..a601acc3b --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationFilter.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationFilter; + +import org.springframework.ai.observation.tracing.TracingHelper; +import org.springframework.util.CollectionUtils; + +/** + * An {@link ObservationFilter} to include the Vector Store search response content in the + * observation. + * + * @author Christian Tzolov + * @author Thomas Vitale + * @since 1.0.0 + */ +public class VectorStoreQueryResponseObservationFilter implements ObservationFilter { + + @Override + public Observation.Context map(Observation.Context context) { + + if (!(context instanceof VectorStoreObservationContext observationContext)) { + return context; + } + + var documents = VectorStoreObservationContentProcessor.documents(observationContext); + + if (!CollectionUtils.isEmpty(documents)) { + observationContext.addHighCardinalityKeyValue( + VectorStoreObservationDocumentation.HighCardinalityKeyNames.DB_VECTOR_QUERY_RESPONSE_DOCUMENTS + .withValue(TracingHelper.concatenateStrings(documents))); + } + + return observationContext; + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandler.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandler.java new file mode 100644 index 000000000..9dbbefc8c --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreQueryResponseObservationHandler.java @@ -0,0 +1,59 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationHandler; +import io.micrometer.tracing.handler.TracingObservationHandler; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; + +import org.springframework.ai.observation.conventions.VectorStoreObservationAttributes; +import org.springframework.ai.observation.conventions.VectorStoreObservationEventNames; +import org.springframework.ai.observation.tracing.TracingHelper; +import org.springframework.util.CollectionUtils; + +/** + * Handler for including the query response content in the observation as a span event. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public class VectorStoreQueryResponseObservationHandler implements ObservationHandler { + + @Override + public void onStop(VectorStoreObservationContext context) { + TracingObservationHandler.TracingContext tracingContext = context + .get(TracingObservationHandler.TracingContext.class); + Span otelSpan = TracingHelper.extractOtelSpan(tracingContext); + + var documents = VectorStoreObservationContentProcessor.documents(context); + + if (!CollectionUtils.isEmpty(documents) && otelSpan != null) { + otelSpan.addEvent(VectorStoreObservationEventNames.CONTENT_QUERY_RESPONSE.value(), Attributes.of( + AttributeKey.stringArrayKey(VectorStoreObservationAttributes.DB_VECTOR_QUERY_CONTENT.value()), + documents)); + } + } + + @Override + public boolean supportsContext(Observation.Context context) { + return context instanceof VectorStoreObservationContext; + } + +} diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/package-info.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/package-info.java new file mode 100644 index 000000000..3fe6863ff --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/observation/package-info.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides classes for observing and storing vector data. + */ +@NonNullApi +@NonNullFields +package org.springframework.ai.vectorstore.observation; + +import org.springframework.lang.NonNullApi; +import org.springframework.lang.NonNullFields; diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/package-info.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/package-info.java new file mode 100644 index 000000000..3edee23fc --- /dev/null +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@NonNullApi +@NonNullFields +package org.springframework.ai.vectorstore; + +import org.springframework.lang.NonNullApi; +import org.springframework.lang.NonNullFields; diff --git a/spring-ai-core/src/main/resources/antlr4/org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 b/spring-ai-vector-store/src/main/resources/antlr4/org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 similarity index 100% rename from spring-ai-core/src/main/resources/antlr4/org/springframework/ai/vectorstore/filter/antlr4/Filters.g4 rename to spring-ai-vector-store/src/main/resources/antlr4/org/springframework/ai/vectorstore/filter/antlr4/Filters.g4