Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix Oracle Vector Store dependencies

Browse Source 
- force override the outdated SpringBoot Oracle version.
 - add auto-config tests.
This commit is contained in:
Christian Tzolov
2024-06-14 07:49:40 +02:00
committed by Christian Tzolov
parent a5d92ec30f
commit baadf4cb53
10 changed files with 242 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
spring-ai-spring-boot-autoconfigure/pom.xml
View File

@@ -109,6 +109,27 @@
<optional>true</optional>
</dependency>
<!-- TEMP: Workaround until Spring Boot updates its Oracle version -->
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc11</artifactId>
<version>${oracle.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ucp</artifactId>
<version>${oracle.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.oracle.database.ha</groupId>
<artifactId>simplefan</artifactId>
<version>${oracle.version}</version>
<optional>true</optional>
</dependency>
<!-- PG Vector Store-->
<dependency>
<groupId>org.springframework.ai</groupId>
@@ -351,6 +372,12 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>oracle-free</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.redis</groupId>
<artifactId>testcontainers-redis</artifactId>

6
spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/oracle/OracleAIVectorSearchStoreAutoConfiguration.java → spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/oracle/OracleVectorStoreAutoConfiguration.java
View File

@@ -32,13 +32,13 @@ import javax.sql.DataSource;
*/
@AutoConfiguration(after = JdbcTemplateAutoConfiguration.class)
@ConditionalOnClass({ OracleVectorStore.class, DataSource.class, JdbcTemplate.class })
@EnableConfigurationProperties(OracleAIVectorSearchStoreProperties.class)
public class OracleAIVectorSearchStoreAutoConfiguration {
@EnableConfigurationProperties(OracleVectorStoreProperties.class)
public class OracleVectorStoreAutoConfiguration {
@Bean
@ConditionalOnMissingBean
public OracleVectorStore vectorStore(JdbcTemplate jdbcTemplate, EmbeddingModel embeddingModel,
OracleAIVectorSearchStoreProperties properties) {
OracleVectorStoreProperties properties) {
return new OracleVectorStore(jdbcTemplate, embeddingModel, properties.getTableName(), properties.getIndexType(),
properties.getDistanceType(), properties.getDimensions(), properties.getSearchAccuracy(),
properties.isInitializeSchema(), properties.isRemoveExistingVectorStoreTable(),

16
spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/oracle/OracleAIVectorSearchStoreProperties.java → spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/oracle/OracleVectorStoreProperties.java
View File

@@ -24,16 +24,16 @@ import static org.springframework.ai.vectorstore.OracleVectorStore.DEFAULT_SEARC
/**
* @author Loïc Lefèvre
*/
@ConfigurationProperties(OracleAIVectorSearchStoreProperties.CONFIG_PREFIX)
public class OracleAIVectorSearchStoreProperties extends CommonVectorStoreProperties {
@ConfigurationProperties(OracleVectorStoreProperties.CONFIG_PREFIX)
public class OracleVectorStoreProperties extends CommonVectorStoreProperties {
public static final String CONFIG_PREFIX = "spring.ai.vectorstore.oracle";
private String tableName = OracleVectorStore.DEFAULT_TABLE_NAME;
private OracleVectorStore.OracleAIVectorSearchIndexType indexType = OracleVectorStore.DEFAULT_INDEX_TYPE;
private OracleVectorStore.OracleVectorStoreIndexType indexType = OracleVectorStore.DEFAULT_INDEX_TYPE;
private OracleVectorStore.OracleAIVectorSearchDistanceType distanceType = OracleVectorStore.DEFAULT_DISTANCE_TYPE;
private OracleVectorStore.OracleVectorStoreDistanceType distanceType = OracleVectorStore.DEFAULT_DISTANCE_TYPE;
private int dimensions = OracleVectorStore.DEFAULT_DIMENSIONS;
@@ -51,19 +51,19 @@ public class OracleAIVectorSearchStoreProperties extends CommonVectorStoreProper
this.tableName = tableName;
}
public OracleVectorStore.OracleAIVectorSearchIndexType getIndexType() {
public OracleVectorStore.OracleVectorStoreIndexType getIndexType() {
return indexType;
}
public void setIndexType(OracleVectorStore.OracleAIVectorSearchIndexType indexType) {
public void setIndexType(OracleVectorStore.OracleVectorStoreIndexType indexType) {
this.indexType = indexType;
}
public OracleVectorStore.OracleAIVectorSearchDistanceType getDistanceType() {
public OracleVectorStore.OracleVectorStoreDistanceType getDistanceType() {
return distanceType;
}
public void setDistanceType(OracleVectorStore.OracleAIVectorSearchDistanceType distanceType) {
public void setDistanceType(OracleVectorStore.OracleVectorStoreDistanceType distanceType) {
this.distanceType = distanceType;
}

2
spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports
View File

@@ -15,7 +15,7 @@ org.springframework.ai.autoconfigure.bedrock.titan.BedrockTitanChatAutoConfigura
org.springframework.ai.autoconfigure.bedrock.titan.BedrockTitanEmbeddingAutoConfiguration
org.springframework.ai.autoconfigure.ollama.OllamaAutoConfiguration
org.springframework.ai.autoconfigure.mistralai.MistralAiAutoConfiguration
org.springframework.ai.autoconfigure.vectorstore.oracle.OracleAIVectorSearchStoreAutoConfiguration
org.springframework.ai.autoconfigure.vectorstore.oracle.OracleVectorStoreAutoConfiguration
org.springframework.ai.autoconfigure.vectorstore.pgvector.PgVectorStoreAutoConfiguration
org.springframework.ai.autoconfigure.vectorstore.pinecone.PineconeVectorStoreAutoConfiguration
org.springframework.ai.autoconfigure.vectorstore.milvus.MilvusVectorStoreAutoConfiguration

110
spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/oracle/OracleVectorStoreAutoConfigurationIT.java Normal file
View File

@@ -0,0 +1,110 @@
/*
* Copyright 2024 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.autoconfigure.vectorstore.oracle;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.ai.document.Document;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.transformers.TransformersEmbeddingModel;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.DefaultResourceLoader;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.oracle.OracleContainer;
import org.testcontainers.utility.MountableFile;
/**
* @author Christian Tzolov
*/
@Testcontainers
public class OracleVectorStoreAutoConfigurationIT {
@Container
static OracleContainer oracle23aiContainer = new OracleContainer("gvenzl/oracle-free:23-slim")
.withCopyFileToContainer(MountableFile.forClasspathResource("/oracle/initialize.sql"),
"/container-entrypoint-initdb.d/initialize.sql");
List<Document> documents = List.of(
new Document(getText("classpath:/test/data/spring.ai.txt"), Map.of("spring", "great")),
new Document(getText("classpath:/test/data/time.shelter.txt")),
new Document(getText("classpath:/test/data/great.depression.txt"), Map.of("depression", "bad")));
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(OracleVectorStoreAutoConfiguration.class))
.withUserConfiguration(Config.class)
.withPropertyValues("spring.ai.vectorstore.oracle.distanceType=COSINE",
// JdbcTemplate configuration
String.format("spring.datasource.url=jdbc:oracle:thin:@//%s:%d/%s", oracle23aiContainer.getHost(),
oracle23aiContainer.getMappedPort(1521), "freepdb1"),
"spring.datasource.username=mlops", "spring.datasource.password=mlops",
"spring.datasource.type=oracle.jdbc.pool.OracleDataSource");
@Test
public void addAndSearch() {
contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
vectorStore.add(documents);
List<Document> results = vectorStore
.similaritySearch(SearchRequest.query("What is Great Depression?").withTopK(1));
assertThat(results).hasSize(1);
Document resultDoc = results.get(0);
assertThat(resultDoc.getId()).isEqualTo(documents.get(2).getId());
assertThat(resultDoc.getMetadata()).containsKeys("depression", "distance");
// Remove all documents from the store
vectorStore.delete(documents.stream().map(doc -> doc.getId()).toList());
results = vectorStore.similaritySearch(SearchRequest.query("Great Depression").withTopK(1));
assertThat(results).hasSize(0);
});
}
public static String getText(String uri) {
var resource = new DefaultResourceLoader().getResource(uri);
try {
return resource.getContentAsString(StandardCharsets.UTF_8);
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
@Configuration(proxyBeanMethods = false)
static class Config {
@Bean
public EmbeddingModel embeddingModel() {
return new TransformersEmbeddingModel();
}
}
}

54
spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/oracle/OracleVectorStorePropertiesTests.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright 2024 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.autoconfigure.vectorstore.oracle;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;
import org.springframework.ai.vectorstore.OracleVectorStore;
import org.springframework.ai.vectorstore.OracleVectorStore.OracleVectorStoreDistanceType;
import org.springframework.ai.vectorstore.OracleVectorStore.OracleVectorStoreIndexType;
/**
* @author Christian Tzolov
*/
public class OracleVectorStorePropertiesTests {
@Test
public void defaultValues() {
var props = new OracleVectorStoreProperties();
assertThat(props.getDimensions()).isEqualTo(OracleVectorStore.DEFAULT_DIMENSIONS);
assertThat(props.getDistanceType()).isEqualTo(OracleVectorStoreDistanceType.COSINE);
assertThat(props.getIndexType()).isEqualTo(OracleVectorStoreIndexType.IVF);
assertThat(props.isRemoveExistingVectorStoreTable()).isFalse();
}
@Test
public void customValues() {
var props = new OracleVectorStoreProperties();
props.setDimensions(1536);
props.setDistanceType(OracleVectorStoreDistanceType.EUCLIDEAN);
props.setIndexType(OracleVectorStoreIndexType.IVF);
props.setRemoveExistingVectorStoreTable(true);
assertThat(props.getDimensions()).isEqualTo(1536);
assertThat(props.getDistanceType()).isEqualTo(OracleVectorStoreDistanceType.EUCLIDEAN);
assertThat(props.getIndexType()).isEqualTo(OracleVectorStoreIndexType.IVF);
assertThat(props.isRemoveExistingVectorStoreTable()).isTrue();
}
}

10
spring-ai-spring-boot-autoconfigure/src/test/resources/oracle/initialize.sql Normal file
View File

@@ -0,0 +1,10 @@
-- Exit on any errors
WHENEVER SQLERROR EXIT SQL.SQLCODE
-- Configure the size of the Vector Pool to 1 GiB.
ALTER SYSTEM SET vector_memory_size=1G SCOPE=SPFILE;
SHUTDOWN ABORT;
STARTUP;
exit;

17
spring-ai-spring-boot-starters/spring-ai-starter-oracle-store/pom.xml
View File

@@ -37,6 +37,23 @@
<artifactId>spring-ai-oracle-store</artifactId>
<version>${project.parent.version}</version>
</dependency>
<!-- TEMP: Workaround until Spring Boot updates its Oracle version -->
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc11</artifactId>
<version>${oracle.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ucp</artifactId>
<version>${oracle.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.database.ha</groupId>
<artifactId>simplefan</artifactId>
<version>${oracle.version}</version>
</dependency>
</dependencies>
</project>

18
vector-stores/spring-ai-oracle-store/src/main/java/org/springframework/ai/vectorstore/OracleVectorStore.java
View File

@@ -44,7 +44,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.springframework.ai.vectorstore.OracleVectorStore.OracleAIVectorSearchDistanceType.DOT;
import static org.springframework.ai.vectorstore.OracleVectorStore.OracleVectorStoreDistanceType.DOT;
import static org.springframework.jdbc.core.StatementCreatorUtils.setParameterValue;
/**
@@ -76,7 +76,7 @@ public class OracleVectorStore implements VectorStore, InitializingBean {
public static final double SIMILARITY_THRESHOLD_EXACT_MATCH = 1.0d;
public enum OracleAIVectorSearchIndexType {
public enum OracleVectorStoreIndexType {
/**
* Performs exact nearest neighbor search.
@@ -123,7 +123,7 @@ public class OracleVectorStore implements VectorStore, InitializingBean {
}
public enum OracleAIVectorSearchDistanceType {
public enum OracleVectorStoreDistanceType {
/**
* Default metric. It calculates the cosine distane between two vectors.
@@ -162,9 +162,9 @@ public class OracleVectorStore implements VectorStore, InitializingBean {
public static final String DEFAULT_TABLE_NAME = "SPRING_AI_VECTORS";
public static final OracleAIVectorSearchIndexType DEFAULT_INDEX_TYPE = OracleAIVectorSearchIndexType.IVF;
public static final OracleVectorStoreIndexType DEFAULT_INDEX_TYPE = OracleVectorStoreIndexType.IVF;
public static final OracleAIVectorSearchDistanceType DEFAULT_DISTANCE_TYPE = OracleAIVectorSearchDistanceType.COSINE;
public static final OracleVectorStoreDistanceType DEFAULT_DISTANCE_TYPE = OracleVectorStoreDistanceType.COSINE;
public static final int DEFAULT_DIMENSIONS = -1;
@@ -189,12 +189,12 @@ public class OracleVectorStore implements VectorStore, InitializingBean {
* Index type used to index the vectors. It can impact performance and database memory
* consumption.
*/
private final OracleAIVectorSearchIndexType indexType;
private final OracleVectorStoreIndexType indexType;
/**
* Distance type to use for computing vector distances.
*/
private final OracleAIVectorSearchDistanceType distanceType;
private final OracleVectorStoreDistanceType distanceType;
/**
* Expected number of dimensions for vectors. Enforcing vector dimensions is very
@@ -217,7 +217,7 @@ public class OracleVectorStore implements VectorStore, InitializingBean {
}
public OracleVectorStore(JdbcTemplate jdbcTemplate, EmbeddingModel embeddingModel, String tableName,
OracleAIVectorSearchIndexType indexType, OracleAIVectorSearchDistanceType distanceType, int dimensions,
OracleVectorStoreIndexType indexType, OracleVectorStoreDistanceType distanceType, int dimensions,
int searchAccuracy, boolean initializeSchema, boolean removeExistingVectorStoreTable,
boolean forcedNormalization) {
if (dimensions != DEFAULT_DIMENSIONS) {
@@ -496,7 +496,7 @@ public class OracleVectorStore implements VectorStore, InitializingBean {
}
else {
if (!forcedNormalization
|| (distanceType != OracleAIVectorSearchDistanceType.COSINE && distanceType != DOT)) {
|| (distanceType != OracleVectorStoreDistanceType.COSINE && distanceType != DOT)) {
throw new RuntimeException(
"Similarity threshold filtering requires all vectors to be normalized, see the forcedNormalization parameter for this Vector store. Also only COSINE and DOT distance types are supported.");
}

6
vector-stores/spring-ai-oracle-store/src/test/java/org/springframework/ai/vectorstore/OracleVectorStoreIT.java
View File

@@ -63,7 +63,7 @@ public class OracleVectorStoreIT {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(TestClient.class)
.withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=COSINE_DISTANCE",
.withPropertyValues("test.spring.ai.vectorstore.oracle.distanceType=COSINE",
"test.spring.ai.vectorstore.oracle.dimensions=384",
// JdbcTemplate configuration
String.format("app.datasource.url=%s", oracle23aiContainer.getJdbcUrl()),
@@ -76,7 +76,7 @@ public class OracleVectorStoreIT {
public static class TestClient {
@Value("${test.spring.ai.vectorstore.oracle.distanceType}")
OracleVectorStore.OracleAIVectorSearchDistanceType distanceType;
OracleVectorStore.OracleVectorStoreDistanceType distanceType;
@Value("${test.spring.ai.vectorstore.oracle.searchAccuracy}")
int searchAccuracy;
@@ -84,7 +84,7 @@ public class OracleVectorStoreIT {
@Bean
public VectorStore vectorStore(JdbcTemplate jdbcTemplate, EmbeddingModel embeddingModel) {
return new OracleVectorStore(jdbcTemplate, embeddingModel, OracleVectorStore.DEFAULT_TABLE_NAME,
OracleVectorStore.OracleAIVectorSearchIndexType.IVF, distanceType, 384, searchAccuracy, true, true,
OracleVectorStore.OracleVectorStoreIndexType.IVF, distanceType, 384, searchAccuracy, true, true,
true);
}