diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/opensearch.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/opensearch.adoc index 66e6e7dfe..2c6fe4f6e 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/opensearch.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/opensearch.adoc @@ -1,50 +1,89 @@ = OpenSearch -This section guides you through setting up the OpenSearch `VectorStore` to store document embeddings and perform similarity searches. +This section walks you through setting up `OpenSearchVectorStore` to store document embeddings and perform similarity searches. -link:https://opensearch.org[OpenSearch] is an open-source search and analytics engine originally forked from Elasticsearch, distributed under the Apache License 2.0. It enhances AI application development by simplifying the integration and management of AI-generated assets. OpenSearch supports vector, lexical, and hybrid search capabilities, leveraging advanced vector database functionalities to facilitate low-latency queries and similarity searches as detailed on the link:https://opensearch.org/platform/search/vector-database.html[vector database page]. This platform is ideal for building scalable AI-driven applications and offers robust tools for data management, fault tolerance, and resource access controls. +link:https://opensearch.org[OpenSearch] is an open-source search and analytics engine originally forked from Elasticsearch, distributed under the Apache License 2.0. It enhances AI application development by simplifying the integration and management of AI-generated assets. OpenSearch supports vector, lexical, and hybrid search capabilities, leveraging advanced vector database functionalities to facilitate low-latency queries and similarity searches as detailed on the link:https://opensearch.org/platform/search/vector-database.html[vector database page]. + +The link:https://opensearch.org/docs/latest/search-plugins/knn/index/[OpenSearch k-NN] functionality allows users to query vector embeddings from large datasets. An embedding is a numerical representation of a data object, such as text, image, audio, or document. Embeddings can be stored in the index and queried using various similarity functions. == Prerequisites * A running OpenSearch instance. The following options are available: ** link:https://opensearch.org/docs/latest/opensearch/install/index/[Self-Managed OpenSearch] ** link:https://docs.aws.amazon.com/opensearch-service/[Amazon OpenSearch Service] -* `EmbeddingModel` instance to compute the document embeddings. Several options are available: -- If required, an API key for the xref:api/embeddings.adoc#available-implementations[EmbeddingModel] to generate the -embeddings stored by the `OpenSearchVectorStore`. +* If required, an API key for the xref:api/embeddings.adoc#available-implementations[EmbeddingModel] to generate the embeddings stored by the `OpenSearchVectorStore`. -== Dependencies +== Auto-configuration -Add the OpenSearch Vector Store dependency to your project: +Spring AI provides Spring Boot auto-configuration for the OpenSearch Vector Store. +To enable it, add the following dependency to your project's Maven `pom.xml` file: -[tabs] -====== -Maven:: -+ [source,xml] ---- org.springframework.ai - spring-ai-opensearch-store + spring-ai-opensearch-store-spring-boot-starter ---- -Gradle:: -+ +or to your Gradle `build.gradle` build file: + [source,groovy] ---- dependencies { - implementation 'org.springframework.ai:spring-ai-opensearch-store' + implementation 'org.springframework.ai:spring-ai-opensearch-store-spring-boot-starter' } ---- -====== TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file. -== Configuration +For Amazon OpenSearch Service, use these dependencies instead: + +[source,xml] +---- + + org.springframework.ai + spring-ai-aws-opensearch-store-spring-boot-starter + +---- + +or for Gradle: + +[source,groovy] +---- +dependencies { + implementation 'org.springframework.ai:spring-ai-aws-opensearch-store-spring-boot-starter' +} +---- + +Please have a look at the list of xref:#_configuration_properties[configuration parameters] for the vector store to learn about the default values and configuration options. + +Additionally, you will need a configured `EmbeddingModel` bean. Refer to the xref:api/embeddings.adoc#available-implementations[EmbeddingModel] section for more information. + +Now you can auto-wire the `OpenSearchVectorStore` as a vector store in your application: + +[source,java] +---- +@Autowired VectorStore vectorStore; + +// ... + +List documents = List.of( + new Document("Spring AI rocks!! Spring AI rocks!! Spring AI rocks!!", Map.of("meta1", "meta1")), + new Document("The World is Big and Salvation Lurks Around the Corner"), + new Document("You walk forward facing the past and you turn back toward the future.", Map.of("meta2", "meta2"))); + +// Add the documents to OpenSearch +vectorStore.add(documents); + +// Retrieve documents similar to a query +List results = vectorStore.similaritySearch(SearchRequest.query("Spring").withTopK(5)); +---- + +=== Configuration Properties To connect to OpenSearch and use the `OpenSearchVectorStore`, you need to provide access details for your instance. -A simple configuration can either be provided via Spring Boot's `application.yml`, +A simple configuration can be provided via Spring Boot's `application.yml`: [source,yaml] ---- @@ -55,146 +94,105 @@ spring: uris: username: password: - indexName: - mappingJson: - aws: + index-name: spring-ai-document-index + initialize-schema: true + similarity-function: cosinesimil + batching-strategy: TOKEN_COUNT + aws: # Only for Amazon OpenSearch Service host: - serviceName: - accessKey: - secretKey: + service-name: + access-key: + secret-key: region: -# API key if needed, e.g. OpenAI - openai: - apiKey: ---- -TIP: Check the list of xref:#_configuration_properties[configuration parameters] to learn about the default values and configuration options. -== Auto-configuration +Properties starting with `spring.ai.vectorstore.opensearch.*` are used to configure the `OpenSearchVectorStore`: -=== Self-Managed OpenSearch +[cols="2,5,1",stripes=even] +|=== +|Property | Description | Default Value -Spring AI provides Spring Boot auto-configuration for the OpenSearch Vector Store. -To enable it, add the following dependency to your project's Maven `pom.xml` or Gradle `build.gradle` build files: +|`spring.ai.vectorstore.opensearch.uris`| URIs of the OpenSearch cluster endpoints | - +|`spring.ai.vectorstore.opensearch.username`| Username for accessing the OpenSearch cluster | - +|`spring.ai.vectorstore.opensearch.password`| Password for the specified username | - +|`spring.ai.vectorstore.opensearch.index-name`| Name of the index to store vectors | `spring-ai-document-index` +|`spring.ai.vectorstore.opensearch.initialize-schema`| Whether to initialize the required schema | `false` +|`spring.ai.vectorstore.opensearch.similarity-function`| The similarity function to use | `cosinesimil` +|`spring.ai.vectorstore.opensearch.batching-strategy`| Strategy for batching documents when calculating embeddings. Options are `TOKEN_COUNT` or `FIXED_SIZE` | `TOKEN_COUNT` +|`spring.ai.vectorstore.opensearch.aws.host`| Hostname of the OpenSearch instance | - +|`spring.ai.vectorstore.opensearch.aws.service-name`| AWS service name | - +|`spring.ai.vectorstore.opensearch.aws.access-key`| AWS access key | - +|`spring.ai.vectorstore.opensearch.aws.secret-key`| AWS secret key | - +|`spring.ai.vectorstore.opensearch.aws.region`| AWS region | - +|=== + +The following similarity functions are available: + +* `cosinesimil` - Default, suitable for most use cases. Measures cosine similarity between vectors. +* `l1` - Manhattan distance between vectors. +* `l2` - Euclidean distance between vectors. +* `linf` - Chebyshev distance between vectors. + +== Manual Configuration + +Instead of using the Spring Boot auto-configuration, you can manually configure the OpenSearch vector store. For this you need to add the `spring-ai-opensearch-store` to your project: -[tabs] -====== -Maven:: -+ [source,xml] ---- org.springframework.ai - spring-ai-opensearch-store-spring-boot-starter + spring-ai-opensearch-store ---- -Gradle:: -+ +or to your Gradle `build.gradle` build file: + [source,groovy] ---- dependencies { - implementation 'org.springframework.ai:spring-ai-opensearch-store-spring-boot-starter' + implementation 'org.springframework.ai:spring-ai-opensearch-store' } ---- -====== - -Then use the `spring.ai.vectorstore.opensearch.*` properties to configure the connection to the self-managed OpenSearch instance. - -=== Amazon OpenSearch Service - -To enable Amazon OpenSearch Service., add the following dependency to your project's Maven `pom.xml` or Gradle `build.gradle` build files: - -[tabs] -====== -Maven:: -+ -[source,xml] ----- - - org.springframework.ai - spring-ai-aws-opensearch-store-spring-boot-starter - ----- - -Gradle:: -+ -[source,groovy] ----- -dependencies { - implementation 'org.springframework.ai:spring-ai-aws-opensearch-store-spring-boot-starter' -} ----- -====== - -Then use the `spring.ai.vectorstore.opensearch.aws.*` properties to configure the connection to the Amazon OpenSearch Service. TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file. -Here is an example of the needed bean: +Create an OpenSearch client bean: [source,java] ---- @Bean -public EmbeddingModel embeddingModel() { - // Can be any other EmbeddingModel implementation - return new OpenAiEmbeddingModel(new OpenAiApi(System.getenv("SPRING_AI_OPENAI_API_KEY"))); +public OpenSearchClient openSearchClient() { + RestClient restClient = RestClient.builder( + HttpHost.create("http://localhost:9200")) + .build(); + + return new OpenSearchClient(new RestClientTransport( + restClient, new JacksonJsonpMapper())); } ---- -Now you can auto-wire the `OpenSearchVectorStore` as a vector store in your application. +Then create the `OpenSearchVectorStore` bean using the builder pattern: [source,java] ---- -@Autowired VectorStore vectorStore; - -// ... - -List documents = List.of( - new Document("Spring AI rocks!! Spring AI rocks!! Spring AI rocks!! Spring AI rocks!! Spring AI rocks!!", Map.of("meta1", "meta1")), - new Document("The World is Big and Salvation Lurks Around the Corner"), - new Document("You walk forward facing the past and you turn back toward the future.", Map.of("meta2", "meta2"))); - -// Add the documents to OpenSearch -vectorStore.add(List.of(document)); - -// Retrieve documents similar to a query -List results = this.vectorStore.similaritySearch(SearchRequest.query("Spring").withTopK(5)); ----- - -=== Configuration properties - -You can use the following properties in your Spring Boot configuration to customize the OpenSearch vector store. - -[cols="2,5,1",stripes=even] -|=== -|Property| Description | Default value - -|`spring.ai.vectorstore.opensearch.uris`| URIs of the OpenSearch cluster endpoints. | - -|`spring.ai.vectorstore.opensearch.username`| Username for accessing the OpenSearch cluster. | - -|`spring.ai.vectorstore.opensearch.password`| Password for the specified username. | - -|`spring.ai.vectorstore.opensearch.indexName`| Name of the default index to be used within the OpenSearch cluster. | `spring-ai-document-index` -|`spring.ai.vectorstore.opensearch.mappingJson`| JSON string defining the mapping for the index; specifies how documents and their -fields are stored and indexed. Refer link:https://opensearch.org/docs/latest/search-plugins/vector-search/[here] for some sample configurations | -{ - "properties":{ - "embedding":{ - "type":"knn_vector", - "dimension":1536 - } - } +@Bean +public VectorStore vectorStore(OpenSearchClient openSearchClient, EmbeddingModel embeddingModel) { + return OpenSearchVectorStore.builder() + .openSearchClient(openSearchClient) + .embeddingModel(embeddingModel) + .index("custom-index") // Optional: defaults to "spring-ai-document-index" + .similarityFunction("l2") // Optional: defaults to "cosinesimil" + .initializeSchema(true) // Optional: defaults to false + .batchingStrategy(new TokenCountBatchingStrategy()) // Optional: defaults to TokenCountBatchingStrategy + .build(); } -|`spring.ai.vectorstore.opensearch.aws.host`| Hostname of the OpenSearch instance. | - -|`spring.ai.vectorstore.opensearch.aws.serviceName`| AWS service name for the OpenSearch instance. | - -|`spring.ai.vectorstore.opensearch.aws.accessKey`| AWS access key for the OpenSearch instance. | - -|`spring.ai.vectorstore.opensearch.aws.secretKey`| AWS secret key for the OpenSearch instance. | - -|`spring.ai.vectorstore.opensearch.aws.region`| AWS region for the OpenSearch instance. | - -|=== -=== Customizing OpenSearch Client Configuration - -In cases where the Spring Boot auto-configured OpenSearchClient with `Apache HttpClient 5 Transport` bean is not what -you want or need, you can still define your own bean. -Please read the link:https://opensearch.org/docs/latest/clients/java/[OpenSearch Java Client Documentation] +// This can be any EmbeddingModel implementation +@Bean +public EmbeddingModel embeddingModel() { + return new OpenAiEmbeddingModel(new OpenAiApi(System.getenv("OPENAI_API_KEY"))); +} +---- == Metadata Filtering @@ -202,34 +200,30 @@ You can leverage the generic, portable xref:api/vectordbs.adoc#metadata-filters[ For example, you can use either the text expression language: -[tabs] -====== -SQL filter syntax:: -+ [source,java] ---- -vectorStore.similaritySearch(SearchRequest.defaults() +vectorStore.similaritySearch( + SearchRequest.defaults() .withQuery("The World") .withTopK(TOP_K) .withSimilarityThreshold(SIMILARITY_THRESHOLD) .withFilterExpression("author in ['john', 'jill'] && 'article_type' == 'blog'")); ---- -`Filter.Expression` DSL:: -+ +or programmatically using the `Filter.Expression` DSL: + [source,java] ---- FilterExpressionBuilder b = new FilterExpressionBuilder(); vectorStore.similaritySearch(SearchRequest.defaults() - .withQuery("The World") - .withTopK(TOP_K) - .withSimilarityThreshold(SIMILARITY_THRESHOLD) - .withFilterExpression(b.and( - b.in("john", "jill"), - b.eq("article_type", "blog")).build())); + .withQuery("The World") + .withTopK(TOP_K) + .withSimilarityThreshold(SIMILARITY_THRESHOLD) + .withFilterExpression(b.and( + b.in("author", "john", "jill"), + b.eq("article_type", "blog")).build())); ---- -====== NOTE: Those (portable) filter expressions get automatically converted into the proprietary OpenSearch link:https://opensearch.org/docs/latest/query-dsl/full-text/query-string/[Query string query]. diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfiguration.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfiguration.java index 47628a30c..28a4e4529 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfiguration.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfiguration.java @@ -40,7 +40,7 @@ import software.amazon.awssdk.regions.Region; import org.springframework.ai.embedding.BatchingStrategy; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.embedding.TokenCountBatchingStrategy; -import org.springframework.ai.vectorstore.OpenSearchVectorStore; +import org.springframework.ai.vectorstore.opensearch.OpenSearchVectorStore; import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention; import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.AutoConfiguration; @@ -78,9 +78,17 @@ public class OpenSearchVectorStoreAutoConfiguration { var indexName = Optional.ofNullable(properties.getIndexName()).orElse(OpenSearchVectorStore.DEFAULT_INDEX_NAME); var mappingJson = Optional.ofNullable(properties.getMappingJson()) .orElse(OpenSearchVectorStore.DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION); - return new OpenSearchVectorStore(indexName, openSearchClient, embeddingModel, mappingJson, - properties.isInitializeSchema(), observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP), - customObservationConvention.getIfAvailable(() -> null), batchingStrategy); + + return OpenSearchVectorStore.builder() + .index(indexName) + .openSearchClient(openSearchClient) + .embeddingModel(embeddingModel) + .mappingJson(mappingJson) + .initializeSchema(properties.isInitializeSchema()) + .observationRegistry(observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP)) + .customObservationConvention(customObservationConvention.getIfAvailable(() -> null)) + .batchingStrategy(batchingStrategy) + .build(); } @Configuration(proxyBeanMethods = false) diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/AwsOpenSearchVectorStoreAutoConfigurationIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/AwsOpenSearchVectorStoreAutoConfigurationIT.java index d3449dc9f..0810c6cab 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/AwsOpenSearchVectorStoreAutoConfigurationIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/AwsOpenSearchVectorStoreAutoConfigurationIT.java @@ -36,7 +36,7 @@ import org.springframework.ai.autoconfigure.retry.SpringAiRetryAutoConfiguration import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.transformers.TransformersEmbeddingModel; -import org.springframework.ai.vectorstore.OpenSearchVectorStore; +import org.springframework.ai.vectorstore.opensearch.OpenSearchVectorStore; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfigurationIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfigurationIT.java index 55022d938..87c774ced 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfigurationIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfigurationIT.java @@ -36,7 +36,7 @@ import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.observation.conventions.VectorStoreProvider; import org.springframework.ai.transformers.TransformersEmbeddingModel; -import org.springframework.ai.vectorstore.OpenSearchVectorStore; +import org.springframework.ai.vectorstore.opensearch.OpenSearchVectorStore; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext; import org.springframework.boot.autoconfigure.AutoConfigurations; diff --git a/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/OpenSearchAiSearchFilterExpressionConverter.java b/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchAiSearchFilterExpressionConverter.java similarity index 98% rename from vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/OpenSearchAiSearchFilterExpressionConverter.java rename to vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchAiSearchFilterExpressionConverter.java index 71022a986..9b5be81e7 100644 --- a/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/OpenSearchAiSearchFilterExpressionConverter.java +++ b/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchAiSearchFilterExpressionConverter.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.vectorstore; +package org.springframework.ai.vectorstore.opensearch; import java.text.ParseException; import java.text.SimpleDateFormat; diff --git a/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/OpenSearchVectorStore.java b/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStore.java similarity index 52% rename from vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/OpenSearchVectorStore.java rename to vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStore.java index 0492ec80f..5a048b29b 100644 --- a/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/OpenSearchVectorStore.java +++ b/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStore.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.vectorstore; +package org.springframework.ai.vectorstore.opensearch; import java.io.IOException; import java.io.StringReader; @@ -37,8 +37,6 @@ import org.opensearch.client.opensearch.core.search.Hit; import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.client.opensearch.indices.CreateIndexResponse; import org.opensearch.client.transport.endpoints.BooleanResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.document.DocumentMetadata; @@ -48,6 +46,8 @@ import org.springframework.ai.embedding.EmbeddingOptionsBuilder; import org.springframework.ai.embedding.TokenCountBatchingStrategy; import org.springframework.ai.observation.conventions.VectorStoreProvider; import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric; +import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder; +import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.filter.Filter; import org.springframework.ai.vectorstore.filter.FilterExpressionConverter; import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore; @@ -57,7 +57,84 @@ import org.springframework.beans.factory.InitializingBean; import org.springframework.util.Assert; /** - * An ObservationVectorStore implementation that stores vectors in OpenSearch. + * OpenSearch-based vector store implementation using OpenSearch's vector search + * capabilities. + * + *

+ * The store uses OpenSearch's k-NN functionality to persist and query vector embeddings + * along with their associated document content and metadata. The implementation supports + * various similarity functions and provides efficient vector search operations. + *

+ * + *

+ * Features: + *

+ *
    + *
  • Automatic schema initialization with configurable index creation
    • + *
  • Support for multiple similarity functions: Cosine, L1, L2, and Linf
    • + *
  • Metadata filtering using OpenSearch query expressions
    • + *
  • Configurable similarity thresholds for search results
    • + *
  • Batch processing support with configurable strategies
    • + *
  • Observation and metrics support through Micrometer
    • + *
+ * + *

+ * Basic usage example: + *

+ * 
{@code
+ * OpenSearchVectorStore vectorStore = OpenSearchVectorStore.builder()
+ *     .openSearchClient(openSearchClient)
+ *     .embeddingModel(embeddingModel)
+ *     .initializeSchema(true)
+ *     .build();
+ *
+ * // Add documents
+ * vectorStore.add(List.of(
+ *     new Document("content1", Map.of("key1", "value1")),
+ *     new Document("content2", Map.of("key2", "value2"))
+ * ));
+ *
+ * // Search with filters
+ * List results = vectorStore.similaritySearch(
+ *     SearchRequest.query("search text")
+ *         .withTopK(5)
+ *         .withSimilarityThreshold(0.7)
+ *         .withFilterExpression("key1 == 'value1'")
+ * );
+ * }
+ * + *

+ * Advanced configuration example: + *

+ * 
{@code
+ * OpenSearchVectorStore vectorStore = OpenSearchVectorStore.builder()
+ *     .openSearchClient(openSearchClient)
+ *     .embeddingModel(embeddingModel)
+ *     .index("custom-index")
+ *     .mappingJson(customMapping)
+ *     .similarityFunction("l2")
+ *     .initializeSchema(true)
+ *     .batchingStrategy(new TokenCountBatchingStrategy())
+ *     .filterExpressionConverter(new CustomFilterExpressionConverter())
+ *     .build();
+ * }
+ * + *

+ * Similarity Functions: + *

+ *
    + *
  • cosinesimil: Default, suitable for most use cases. Measures cosine similarity + * between vectors.
    • + *
  • l1: Manhattan distance between vectors.
    • + *
  • l2: Euclidean distance between vectors.
    • + *
  • linf: Chebyshev distance between vectors.
    • + *
+ * + *

+ * For more information about available similarity functions, see: OpenSearch + * KNN Spaces + *

* * @author Jemin Huh * @author Soby Chacko @@ -83,10 +160,6 @@ public class OpenSearchVectorStore extends AbstractObservationVectorStore implem } """; - private static final Logger logger = LoggerFactory.getLogger(OpenSearchVectorStore.class); - - private final EmbeddingModel embeddingModel; - private final OpenSearchClient openSearchClient; private final String index; @@ -101,40 +174,106 @@ public class OpenSearchVectorStore extends AbstractObservationVectorStore implem private String similarityFunction; + /** + * Creates a new OpenSearchVectorStore with default mapping and collection name. + * @deprecated Use {@link #builder()} instead + * @param openSearchClient The OpenSearch client + * @param embeddingModel The embedding model to use + * @param initializeSchema Whether to initialize the schema + * @since 1.0.0 + */ + @Deprecated(since = "1.0.0-M5", forRemoval = true) public OpenSearchVectorStore(OpenSearchClient openSearchClient, EmbeddingModel embeddingModel, boolean initializeSchema) { this(openSearchClient, embeddingModel, DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION, initializeSchema); } + /** + * Creates a new OpenSearchVectorStore with custom mapping. + * @deprecated Use {@link #builder()} instead + * @param openSearchClient The OpenSearch client + * @param embeddingModel The embedding model to use + * @param mappingJson The JSON mapping for the index + * @param initializeSchema Whether to initialize the schema + * @since 1.0.0 + */ + @Deprecated(since = "1.0.0-M5", forRemoval = true) public OpenSearchVectorStore(OpenSearchClient openSearchClient, EmbeddingModel embeddingModel, String mappingJson, boolean initializeSchema) { this(DEFAULT_INDEX_NAME, openSearchClient, embeddingModel, mappingJson, initializeSchema); } + /** + * Creates a new OpenSearchVectorStore with custom index name and mapping. + * @deprecated Use {@link #builder()} instead + * @param index The name of the index + * @param openSearchClient The OpenSearch client + * @param embeddingModel The embedding model to use + * @param mappingJson The JSON mapping for the index + * @param initializeSchema Whether to initialize the schema + * @since 1.0.0 + */ + @Deprecated(since = "1.0.0-M5", forRemoval = true) public OpenSearchVectorStore(String index, OpenSearchClient openSearchClient, EmbeddingModel embeddingModel, String mappingJson, boolean initializeSchema) { this(index, openSearchClient, embeddingModel, mappingJson, initializeSchema, ObservationRegistry.NOOP, null, new TokenCountBatchingStrategy()); } + /** + * Creates a new OpenSearchVectorStore with all configuration options. + * @deprecated Use {@link #builder()} instead + * @param index The name of the index + * @param openSearchClient The OpenSearch client + * @param embeddingModel The embedding model to use + * @param mappingJson The JSON mapping for the index + * @param initializeSchema Whether to initialize the schema + * @param observationRegistry The observation registry for metrics + * @param customObservationConvention Custom observation convention + * @param batchingStrategy The strategy for batching operations + * @since 1.0.0 + */ + @Deprecated(since = "1.0.0-M5", forRemoval = true) public OpenSearchVectorStore(String index, OpenSearchClient openSearchClient, EmbeddingModel embeddingModel, String mappingJson, boolean initializeSchema, ObservationRegistry observationRegistry, VectorStoreObservationConvention customObservationConvention, BatchingStrategy batchingStrategy) { - super(observationRegistry, customObservationConvention); + this(builder().openSearchClient(openSearchClient) + .embeddingModel(embeddingModel) + .index(index) + .mappingJson(mappingJson) + .initializeSchema(initializeSchema) + .observationRegistry(observationRegistry) + .customObservationConvention(customObservationConvention) + .batchingStrategy(batchingStrategy)); + } - Objects.requireNonNull(embeddingModel, "RestClient must not be null"); - Objects.requireNonNull(embeddingModel, "EmbeddingModel must not be null"); - this.openSearchClient = openSearchClient; - this.embeddingModel = embeddingModel; - this.index = index; - this.mappingJson = mappingJson; - this.filterExpressionConverter = new OpenSearchAiSearchFilterExpressionConverter(); + /** + * Creates a new OpenSearchVectorStore using the builder pattern. + * @param builder The configured builder instance + */ + protected OpenSearchVectorStore(OpenSearchBuilder builder) { + super(builder); + + Assert.notNull(builder.openSearchClient, "OpenSearchClient must not be null"); + + this.openSearchClient = builder.openSearchClient; + this.index = builder.index; + this.mappingJson = builder.mappingJson; + this.filterExpressionConverter = builder.filterExpressionConverter; // the potential functions for vector fields at // https://opensearch.org/docs/latest/search-plugins/knn/approximate-knn/#spaces - this.similarityFunction = COSINE_SIMILARITY_FUNCTION; - this.initializeSchema = initializeSchema; - this.batchingStrategy = batchingStrategy; + this.similarityFunction = builder.similarityFunction; + this.initializeSchema = builder.initializeSchema; + this.batchingStrategy = builder.batchingStrategy; + } + + /** + * Creates a new builder instance for configuring an OpenSearchVectorStore. + * @return A new OpenSearchBuilder instance + */ + public static OpenSearchBuilder builder() { + return new OpenSearchBuilder(); } public OpenSearchVectorStore withSimilarityFunction(String similarityFunction) { @@ -306,4 +445,132 @@ public class OpenSearchVectorStore extends AbstractObservationVectorStore implem public record OpenSearchDocument(String id, String content, Map metadata, float[] embedding) { } + /** + * Builder class for creating OpenSearchVectorStore instances. + */ + public static class OpenSearchBuilder extends AbstractVectorStoreBuilder { + + private OpenSearchClient openSearchClient; + + private String index = DEFAULT_INDEX_NAME; + + private String mappingJson = DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION; + + private boolean initializeSchema = false; + + private BatchingStrategy batchingStrategy = new TokenCountBatchingStrategy(); + + private FilterExpressionConverter filterExpressionConverter = new OpenSearchAiSearchFilterExpressionConverter(); + + private String similarityFunction = COSINE_SIMILARITY_FUNCTION; + + /** + * Sets the OpenSearch client. + * @param openSearchClient The OpenSearch client to use + * @return The builder instance + * @throws IllegalArgumentException if openSearchClient is null + */ + public OpenSearchBuilder openSearchClient(OpenSearchClient openSearchClient) { + Assert.notNull(openSearchClient, "OpenSearchClient must not be null"); + this.openSearchClient = openSearchClient; + return this; + } + + /** + * Sets the embedding model. + * @param embeddingModel The embedding model to use + * @return The builder instance + * @throws IllegalArgumentException if embeddingModel is null + */ + public OpenSearchBuilder embeddingModel(EmbeddingModel embeddingModel) { + Assert.notNull(embeddingModel, "EmbeddingModel must not be null"); + this.embeddingModel = embeddingModel; + return this; + } + + /** + * Sets the index name. + * @param index The name of the index to use + * @return The builder instance + * @throws IllegalArgumentException if index is null or empty + */ + public OpenSearchBuilder index(String index) { + Assert.hasText(index, "index must not be null or empty"); + this.index = index; + return this; + } + + /** + * Sets the JSON mapping for the index. + * @param mappingJson The JSON mapping to use + * @return The builder instance + * @throws IllegalArgumentException if mappingJson is null or empty + */ + public OpenSearchBuilder mappingJson(String mappingJson) { + Assert.hasText(mappingJson, "mappingJson must not be null or empty"); + this.mappingJson = mappingJson; + return this; + } + + /** + * Sets whether to initialize the schema. + * @param initializeSchema true to initialize schema, false otherwise + * @return The builder instance + */ + public OpenSearchBuilder initializeSchema(boolean initializeSchema) { + this.initializeSchema = initializeSchema; + return this; + } + + /** + * Sets the batching strategy. + * @param batchingStrategy The batching strategy to use + * @return The builder instance + * @throws IllegalArgumentException if batchingStrategy is null + */ + public OpenSearchBuilder batchingStrategy(BatchingStrategy batchingStrategy) { + Assert.notNull(batchingStrategy, "batchingStrategy must not be null"); + this.batchingStrategy = batchingStrategy; + return this; + } + + /** + * Sets the filter expression converter. + * @param converter The filter expression converter to use + * @return The builder instance + * @throws IllegalArgumentException if converter is null + */ + public OpenSearchBuilder filterExpressionConverter(FilterExpressionConverter converter) { + Assert.notNull(converter, "filterExpressionConverter must not be null"); + this.filterExpressionConverter = converter; + return this; + } + + /** + * Sets the similarity function for vector comparison. See + * https://opensearch.org/docs/latest/search-plugins/knn/approximate-knn/#spaces + * for available functions. + * @param similarityFunction The similarity function to use + * @return The builder instance + * @throws IllegalArgumentException if similarityFunction is null or empty + */ + public OpenSearchBuilder similarityFunction(String similarityFunction) { + Assert.hasText(similarityFunction, "similarityFunction must not be null or empty"); + this.similarityFunction = similarityFunction; + return this; + } + + /** + * Builds a new OpenSearchVectorStore instance with the configured properties. + * @return A new OpenSearchVectorStore instance + * @throws IllegalStateException if the builder is in an invalid state + */ + @Override + public OpenSearchVectorStore build() { + validate(); + return new OpenSearchVectorStore(this); + } + + } + } diff --git a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchAiSearchFilterExpressionConverterTest.java b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchAiSearchFilterExpressionConverterTest.java similarity index 99% rename from vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchAiSearchFilterExpressionConverterTest.java rename to vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchAiSearchFilterExpressionConverterTest.java index 77e2a95a0..ba511a850 100644 --- a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchAiSearchFilterExpressionConverterTest.java +++ b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchAiSearchFilterExpressionConverterTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.vectorstore; +package org.springframework.ai.vectorstore.opensearch; import java.util.Date; import java.util.List; diff --git a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchImage.java b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchImage.java similarity index 94% rename from vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchImage.java rename to vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchImage.java index 42c8d9b9c..99bf315e3 100644 --- a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchImage.java +++ b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchImage.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.vectorstore; +package org.springframework.ai.vectorstore.opensearch; import org.testcontainers.utility.DockerImageName; diff --git a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreIT.java b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreIT.java similarity index 94% rename from vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreIT.java rename to vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreIT.java index 5a4a5b20a..e650efee8 100644 --- a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreIT.java +++ b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreIT.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.vectorstore; +package org.springframework.ai.vectorstore.opensearch; import java.io.IOException; import java.net.URISyntaxException; @@ -47,6 +47,8 @@ import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.openai.OpenAiEmbeddingModel; import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -396,9 +398,13 @@ class OpenSearchVectorStoreIT { @Qualifier("vectorStore") public OpenSearchVectorStore vectorStore(EmbeddingModel embeddingModel) { try { - return new OpenSearchVectorStore(new OpenSearchClient(ApacheHttpClient5TransportBuilder - .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) - .build()), embeddingModel, true); + return OpenSearchVectorStore.builder() + .openSearchClient(new OpenSearchClient(ApacheHttpClient5TransportBuilder + .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) + .build())) + .embeddingModel(embeddingModel) + .initializeSchema(true) + .build(); } catch (URISyntaxException e) { throw new RuntimeException(e); @@ -409,12 +415,15 @@ class OpenSearchVectorStoreIT { @Qualifier("anotherVectorStore") public OpenSearchVectorStore anotherVectorStore(EmbeddingModel embeddingModel) { try { - return new OpenSearchVectorStore("another_index", - new OpenSearchClient(ApacheHttpClient5TransportBuilder - .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) - .build()), - embeddingModel, OpenSearchVectorStore.DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION, - true); + return OpenSearchVectorStore.builder() + .index("another_index") + .openSearchClient(new OpenSearchClient(ApacheHttpClient5TransportBuilder + .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) + .build())) + .embeddingModel(embeddingModel) + .mappingJson(OpenSearchVectorStore.DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION) + .initializeSchema(true) + .build(); } catch (URISyntaxException e) { throw new RuntimeException(e); diff --git a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreObservationIT.java b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreObservationIT.java similarity index 92% rename from vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreObservationIT.java rename to vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreObservationIT.java index 69a36ce52..cba4321fb 100644 --- a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreObservationIT.java +++ b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreObservationIT.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.vectorstore; +package org.springframework.ai.vectorstore.opensearch; import java.io.IOException; import java.net.URISyntaxException; @@ -47,6 +47,8 @@ import org.springframework.ai.observation.conventions.VectorStoreProvider; import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric; import org.springframework.ai.openai.OpenAiEmbeddingModel; import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.VectorStore; import org.springframework.ai.vectorstore.observation.DefaultVectorStoreObservationConvention; import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.HighCardinalityKeyNames; import org.springframework.ai.vectorstore.observation.VectorStoreObservationDocumentation.LowCardinalityKeyNames; @@ -205,12 +207,18 @@ public class OpenSearchVectorStoreObservationIT { public OpenSearchVectorStore vectorStore(EmbeddingModel embeddingModel, ObservationRegistry observationRegistry) { try { - return new OpenSearchVectorStore(OpenSearchVectorStore.DEFAULT_INDEX_NAME, - new OpenSearchClient(ApacheHttpClient5TransportBuilder - .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) - .build()), - embeddingModel, OpenSearchVectorStore.DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION, true, - observationRegistry, null, new TokenCountBatchingStrategy()); + return OpenSearchVectorStore.builder() + .index(OpenSearchVectorStore.DEFAULT_INDEX_NAME) + .openSearchClient(new OpenSearchClient(ApacheHttpClient5TransportBuilder + .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) + .build())) + .embeddingModel(embeddingModel) + .mappingJson(OpenSearchVectorStore.DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION) + .initializeSchema(true) + .observationRegistry(observationRegistry) + .customObservationConvention(null) + .batchingStrategy(new TokenCountBatchingStrategy()) + .build(); } catch (URISyntaxException e) { throw new RuntimeException(e); diff --git a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreWithOllamaIT.java b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreWithOllamaIT.java similarity index 89% rename from vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreWithOllamaIT.java rename to vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreWithOllamaIT.java index ef71b5cf4..5ef309c83 100644 --- a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/OpenSearchVectorStoreWithOllamaIT.java +++ b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreWithOllamaIT.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.vectorstore; +package org.springframework.ai.vectorstore.opensearch; import java.io.IOException; import java.net.URISyntaxException; @@ -46,6 +46,8 @@ import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.ai.ollama.management.ModelManagementOptions; import org.springframework.ai.ollama.management.OllamaModelManager; import org.springframework.ai.ollama.management.PullModelStrategy; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.runner.ApplicationContextRunner; @@ -167,9 +169,13 @@ class OpenSearchVectorStoreWithOllamaIT { @Qualifier("vectorStore") public OpenSearchVectorStore vectorStore(EmbeddingModel embeddingModel) { try { - return new OpenSearchVectorStore(new OpenSearchClient(ApacheHttpClient5TransportBuilder - .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) - .build()), embeddingModel, true); + return OpenSearchVectorStore.builder() + .openSearchClient(new OpenSearchClient(ApacheHttpClient5TransportBuilder + .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) + .build())) + .embeddingModel(embeddingModel) + .initializeSchema(true) + .build(); } catch (URISyntaxException e) { throw new RuntimeException(e); @@ -180,12 +186,15 @@ class OpenSearchVectorStoreWithOllamaIT { @Qualifier("anotherVectorStore") public OpenSearchVectorStore anotherVectorStore(EmbeddingModel embeddingModel) { try { - return new OpenSearchVectorStore("another_index", - new OpenSearchClient(ApacheHttpClient5TransportBuilder - .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) - .build()), - embeddingModel, OpenSearchVectorStore.DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION, - true); + return OpenSearchVectorStore.builder() + .index("another_index") + .openSearchClient(new OpenSearchClient(ApacheHttpClient5TransportBuilder + .builder(HttpHost.create(opensearchContainer.getHttpHostAddress())) + .build())) + .embeddingModel(embeddingModel) + .mappingJson(OpenSearchVectorStore.DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION) + .initializeSchema(true) + .build(); } catch (URISyntaxException e) { throw new RuntimeException(e);