From 2c7dc3d6f1a59107bf8c18b77e3ef0d48a5e3348 Mon Sep 17 00:00:00 2001 From: Christian Tzolov Date: Tue, 18 Jun 2024 07:52:46 +0200 Subject: [PATCH] Pinecone: Add configurable content and distance metadata fields (#882) - Add `withContentFieldName` and `withDistanceMetadataFieldName` to Pinecone Config Builder - Add `spring.ai.vectorstore.pinecone.contentFieldName` and `spring.ai.vectorstore.pinecone.distanceMetadataFieldName` properties to the auto-config - Default content field name: "document_content", distance field: "distance" - Update tests and docs Resolves #882 --- .../src/main/antora/modules/ROOT/nav.adoc | 2 +- .../ROOT/pages/api/vectordbs/pinecone.adoc | 4 ++ .../PineconeVectorStoreAutoConfiguration.java | 2 + .../PineconeVectorStoreProperties.java | 21 ++++++++ ...ineconeVectorStoreAutoConfigurationIT.java | 19 +++---- .../PineconeVectorStorePropertiesTests.java | 11 +++- .../ai/vectorstore/PineconeVectorStore.java | 53 ++++++++++++++++--- .../ai/vectorstore/PineconeVectorStoreIT.java | 3 ++ 8 files changed, 96 insertions(+), 19 deletions(-) diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc index 7bf390257..b554b8959 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc @@ -63,7 +63,7 @@ *** xref:api/vectordbs/mongodb.adoc[] *** xref:api/vectordbs/neo4j.adoc[] *** xref:api/vectordbs/opensearch.adoc[] -*** xref:api/vectordbs/oracle.adoc[Oracle DB AI Vector Search] +*** xref:api/vectordbs/oracle.adoc[Oracle] *** xref:api/vectordbs/pgvector.adoc[] *** xref:api/vectordbs/pinecone.adoc[] *** xref:api/vectordbs/qdrant.adoc[] diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/pinecone.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/pinecone.adoc index 71d614600..b798b6628 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/pinecone.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/pinecone.adoc @@ -111,6 +111,9 @@ You can use the following properties in your Spring Boot configuration to custom |`spring.ai.vectorstore.pinecone.project-id`| Pinecone project ID | - |`spring.ai.vectorstore.pinecone.index-name`| Pinecone index name | - |`spring.ai.vectorstore.pinecone.namespace`| Pinecone namespace | - +|`spring.ai.vectorstore.pinecone.namespace`| Pinecone namespace | - +|`spring.ai.vectorstore.pinecone.content-field-name`| Pinecone metadata field name used to store the origina text content. | `document_content` +|`spring.ai.vectorstore.pinecone.distance-metadata-field-name`| Pinecone metadata field name used to store the computed distance. | `distance` |`spring.ai.vectorstore.pinecone.server-side-timeout`| | 20 sec. |=== @@ -193,6 +196,7 @@ public PineconeVectorStoreConfig pineconeVectorStoreConfig() { .withProjectId("89309e6") .withIndexName("spring-ai-test-index") .withNamespace("") // the free tier doesn't support namespaces. + .withContentFieldName("my_content") // optional field to store the original content. Defaults to `document_content` .build(); } ---- diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreAutoConfiguration.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreAutoConfiguration.java index 2da22bb6f..7356e1b06 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreAutoConfiguration.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreAutoConfiguration.java @@ -42,6 +42,8 @@ public class PineconeVectorStoreAutoConfiguration { .withProjectId(properties.getProjectId()) .withIndexName(properties.getIndexName()) .withNamespace(properties.getNamespace()) + .withContentFieldName(properties.getContentFieldName()) + .withDistanceMetadataFieldName(properties.getDistanceMetadataFieldName()) .withServerSideTimeout(properties.getServerSideTimeout()) .build(); diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreProperties.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreProperties.java index 8016e24a6..3ba28c228 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreProperties.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreProperties.java @@ -17,6 +17,7 @@ package org.springframework.ai.autoconfigure.vectorstore.pinecone; import java.time.Duration; +import org.springframework.ai.vectorstore.PineconeVectorStore; import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -37,6 +38,10 @@ public class PineconeVectorStoreProperties { private String namespace = ""; + private String contentFieldName = PineconeVectorStore.CONTENT_FIELD_NAME; + + private String distanceMetadataFieldName = PineconeVectorStore.DISTANCE_METADATA_FIELD_NAME; + private Duration serverSideTimeout = Duration.ofSeconds(20); public String getApiKey() { @@ -87,4 +92,20 @@ public class PineconeVectorStoreProperties { this.serverSideTimeout = serverSideTimeout; } + public String getContentFieldName() { + return this.contentFieldName; + } + + public void setContentFieldName(String contentFieldName) { + this.contentFieldName = contentFieldName; + } + + public String getDistanceMetadataFieldName() { + return this.distanceMetadataFieldName; + } + + public void setDistanceMetadataFieldName(String distanceMetadataFieldName) { + this.distanceMetadataFieldName = distanceMetadataFieldName; + } + } diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreAutoConfigurationIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreAutoConfigurationIT.java index e7cf1c3d9..90a5e965f 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreAutoConfigurationIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStoreAutoConfigurationIT.java @@ -15,32 +15,31 @@ */ package org.springframework.ai.autoconfigure.vectorstore.pinecone; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.hasSize; + import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import org.awaitility.Awaitility; -import java.time.Duration; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; - import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.transformers.TransformersEmbeddingModel; +import org.springframework.ai.vectorstore.PineconeVectorStore; import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.ai.vectorstore.VectorStore; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.DefaultResourceLoader; -import static org.assertj.core.api.Assertions.assertThat; -import static org.hamcrest.Matchers.hasSize; - /** * @author Christian Tzolov */ @@ -68,7 +67,9 @@ public class PineconeVectorStoreAutoConfigurationIT { .withPropertyValues("spring.ai.vectorstore.pinecone.apiKey=" + System.getenv("PINECONE_API_KEY"), "spring.ai.vectorstore.pinecone.environment=gcp-starter", "spring.ai.vectorstore.pinecone.projectId=814621f", - "spring.ai.vectorstore.pinecone.indexName=spring-ai-test-index"); + "spring.ai.vectorstore.pinecone.indexName=spring-ai-test-index", + "spring.ai.vectorstore.pinecone.contentFieldName=customContentField", + "spring.ai.vectorstore.pinecone.distanceMetadataFieldName=customDistanceField"); @BeforeAll public static void beforeAll() { @@ -82,7 +83,7 @@ public class PineconeVectorStoreAutoConfigurationIT { contextRunner.run(context -> { - VectorStore vectorStore = context.getBean(VectorStore.class); + PineconeVectorStore vectorStore = context.getBean(PineconeVectorStore.class); vectorStore.add(documents); @@ -98,7 +99,7 @@ public class PineconeVectorStoreAutoConfigurationIT { assertThat(resultDoc.getContent()).contains( "Spring AI provides abstractions that serve as the foundation for developing AI applications."); assertThat(resultDoc.getMetadata()).hasSize(2); - assertThat(resultDoc.getMetadata()).containsKeys("spring", "distance"); + assertThat(resultDoc.getMetadata()).containsKeys("spring", "customDistanceField"); // Remove all documents from the store vectorStore.delete(documents.stream().map(doc -> doc.getId()).toList()); diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStorePropertiesTests.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStorePropertiesTests.java index 4baeac0e1..ce006a438 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStorePropertiesTests.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/pinecone/PineconeVectorStorePropertiesTests.java @@ -15,11 +15,12 @@ */ package org.springframework.ai.autoconfigure.vectorstore.pinecone; +import static org.assertj.core.api.Assertions.assertThat; + import java.time.Duration; import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; +import org.springframework.ai.vectorstore.PineconeVectorStore; /** * @author Christian Tzolov @@ -35,6 +36,8 @@ public class PineconeVectorStorePropertiesTests { assertThat(props.getProjectId()).isNull(); assertThat(props.getIndexName()).isNull(); assertThat(props.getServerSideTimeout()).isEqualTo(Duration.ofSeconds(20)); + assertThat(props.getContentFieldName()).isEqualTo(PineconeVectorStore.CONTENT_FIELD_NAME); + assertThat(props.getDistanceMetadataFieldName()).isEqualTo(PineconeVectorStore.DISTANCE_METADATA_FIELD_NAME); } @Test @@ -46,6 +49,8 @@ public class PineconeVectorStorePropertiesTests { props.setNamespace("namespace"); props.setProjectId("project"); props.setServerSideTimeout(Duration.ofSeconds(60)); + props.setContentFieldName("article"); + props.setDistanceMetadataFieldName("distance2"); assertThat(props.getEnvironment()).isEqualTo("env"); assertThat(props.getNamespace()).isEqualTo("namespace"); @@ -53,6 +58,8 @@ public class PineconeVectorStorePropertiesTests { assertThat(props.getProjectId()).isEqualTo("project"); assertThat(props.getIndexName()).isEqualTo("index"); assertThat(props.getServerSideTimeout()).isEqualTo(Duration.ofSeconds(60)); + assertThat(props.getContentFieldName()).isEqualTo("article"); + assertThat(props.getDistanceMetadataFieldName()).isEqualTo("distance2"); } } diff --git a/vector-stores/spring-ai-pinecone-store/src/main/java/org/springframework/ai/vectorstore/PineconeVectorStore.java b/vector-stores/spring-ai-pinecone-store/src/main/java/org/springframework/ai/vectorstore/PineconeVectorStore.java index 992e8c859..cebbb5cc6 100644 --- a/vector-stores/spring-ai-pinecone-store/src/main/java/org/springframework/ai/vectorstore/PineconeVectorStore.java +++ b/vector-stores/spring-ai-pinecone-store/src/main/java/org/springframework/ai/vectorstore/PineconeVectorStore.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.Struct; import com.google.protobuf.Value; @@ -51,9 +52,9 @@ import org.springframework.util.StringUtils; */ public class PineconeVectorStore implements VectorStore { - private static final String CONTENT_FIELD_NAME = "document_content"; + public static final String CONTENT_FIELD_NAME = "document_content"; - private static final String DISTANCE_METADATA_FIELD_NAME = "distance"; + public static final String DISTANCE_METADATA_FIELD_NAME = "distance"; public final FilterExpressionConverter filterExpressionConverter = new PineconeFilterExpressionConverter(); @@ -63,6 +64,10 @@ public class PineconeVectorStore implements VectorStore { private final String pineconeNamespace; + private final String pineconeContentFieldName; + + private final String pineconeDistanceMetadataFieldName; + private final ObjectMapper objectMapper; /** @@ -74,6 +79,10 @@ public class PineconeVectorStore implements VectorStore { // Leave the namespace empty (e.g. "") for the free tier. private final String namespace; + private final String contentFieldName; + + private final String distanceMetadataFieldName; + private final PineconeConnectionConfig connectionConfig; private final PineconeClientConfig clientConfig; @@ -90,6 +99,9 @@ public class PineconeVectorStore implements VectorStore { */ public PineconeVectorStoreConfig(Builder builder) { this.namespace = builder.namespace; + this.contentFieldName = builder.contentFieldName; + this.distanceMetadataFieldName = builder.distanceMetadataFieldName; + // this.defaultSimilarityTopK = builder.defaultSimilarityTopK; this.connectionConfig = new PineconeConnectionConfig().withIndexName(builder.indexName); this.clientConfig = new PineconeClientConfig().withApiKey(builder.apiKey) @@ -127,6 +139,10 @@ public class PineconeVectorStore implements VectorStore { // The free-tier (gcp-starter) doesn't support Namespaces! private String namespace = ""; + private String contentFieldName = CONTENT_FIELD_NAME; + + private String distanceMetadataFieldName = DISTANCE_METADATA_FIELD_NAME; + /** * Optional server-side timeout in seconds for all operations. Default: 20 * seconds. @@ -187,6 +203,26 @@ public class PineconeVectorStore implements VectorStore { return this; } + /** + * Content field name. + * @param contentFieldName content field name to use. + * @return this builder. + */ + public Builder withContentFieldName(String contentFieldName) { + this.contentFieldName = contentFieldName; + return this; + } + + /** + * Distance metadata field name. + * @param distanceMetadataFieldName distance metadata field name to use. + * @return this builder. + */ + public Builder withDistanceMetadataFieldName(String distanceMetadataFieldName) { + this.distanceMetadataFieldName = distanceMetadataFieldName; + return this; + } + /** * Pinecone server side timeout. * @param serverSideTimeout server timeout to use. @@ -219,6 +255,8 @@ public class PineconeVectorStore implements VectorStore { this.embeddingModel = embeddingModel; this.pineconeNamespace = config.namespace; + this.pineconeContentFieldName = config.contentFieldName; + this.pineconeDistanceMetadataFieldName = config.distanceMetadataFieldName; this.pineconeConnection = new PineconeClient(config.clientConfig).connect(config.connectionConfig); this.objectMapper = new ObjectMapper(); } @@ -269,7 +307,7 @@ public class PineconeVectorStore implements VectorStore { JsonFormat.parser() .ignoringUnknownFields() .merge(this.objectMapper.writeValueAsString(document.getMetadata()), structBuilder); - structBuilder.putFields(CONTENT_FIELD_NAME, contentValue(document)); + structBuilder.putFields(this.pineconeContentFieldName, contentValue(document)); return structBuilder.build(); } catch (Exception e) { @@ -341,9 +379,9 @@ public class PineconeVectorStore implements VectorStore { .map(scoredVector -> { var id = scoredVector.getId(); Struct metadataStruct = scoredVector.getMetadata(); - var content = metadataStruct.getFieldsOrThrow(CONTENT_FIELD_NAME).getStringValue(); + var content = metadataStruct.getFieldsOrThrow(this.pineconeContentFieldName).getStringValue(); Map metadata = extractMetadata(metadataStruct); - metadata.put(DISTANCE_METADATA_FIELD_NAME, 1 - scoredVector.getScore()); + metadata.put(this.pineconeDistanceMetadataFieldName, 1 - scoredVector.getScore()); return new Document(id, content, metadata); }) .toList(); @@ -374,8 +412,9 @@ public class PineconeVectorStore implements VectorStore { private Map extractMetadata(Struct metadataStruct) { try { String json = JsonFormat.printer().print(metadataStruct); - Map metadata = this.objectMapper.readValue(json, Map.class); - metadata.remove(CONTENT_FIELD_NAME); + Map metadata = this.objectMapper.readValue(json, new TypeReference>() { + }); + metadata.remove(this.pineconeContentFieldName); return metadata; } catch (Exception e) { diff --git a/vector-stores/spring-ai-pinecone-store/src/test/java/org/springframework/ai/vectorstore/PineconeVectorStoreIT.java b/vector-stores/spring-ai-pinecone-store/src/test/java/org/springframework/ai/vectorstore/PineconeVectorStoreIT.java index ca7336f12..a0f8a257c 100644 --- a/vector-stores/spring-ai-pinecone-store/src/test/java/org/springframework/ai/vectorstore/PineconeVectorStoreIT.java +++ b/vector-stores/spring-ai-pinecone-store/src/test/java/org/springframework/ai/vectorstore/PineconeVectorStoreIT.java @@ -60,6 +60,8 @@ public class PineconeVectorStoreIT { // NOTE: Leave it empty as for free tier as later doesn't support namespaces. private static final String PINECONE_NAMESPACE = ""; + private static final String CUSTOM_CONTENT_FIELD_NAME = "article"; + List documents = List.of( new Document("1", getText("classpath:/test/data/spring.ai.txt"), Map.of("meta1", "meta1")), new Document("2", getText("classpath:/test/data/time.shelter.txt"), Map.of()), @@ -283,6 +285,7 @@ public class PineconeVectorStoreIT { .withProjectId(PINECONE_PROJECT_ID) .withIndexName(PINECONE_INDEX_NAME) .withNamespace(PINECONE_NAMESPACE) + .withContentFieldName(CUSTOM_CONTENT_FIELD_NAME) .build(); }