Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use Testcontainers Vector DBs modules in autoconfiguartion module

Browse Source
This commit is contained in:
Eddú Meléndez
2024-03-06 22:14:08 -06:00
committed by Christian Tzolov
parent 476164b6e3
commit 879f992394
6 changed files with 40 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pom.xml
View File

@@ -170,6 +170,9 @@
<qdrant.version>1.7.1</qdrant.version>
<typesense.version>0.5.0</typesense.version>
<!-- testing dependencies -->
<testcontainers.version>1.19.7</testcontainers.version>
<!-- documentation dependencies -->
<io.spring.maven.antora-version>0.0.4</io.spring.maven.antora-version>
<asciidoctorj-pdf.version>1.6.2</asciidoctorj-pdf.version> <!-- FIXME build failure with version 2.3.9 -->

21
spring-ai-spring-boot-autoconfigure/pom.xml
View File

@@ -406,15 +406,36 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>chromadb</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>milvus</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>qdrant</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>elasticsearch</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>weaviate</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>

10
spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/chroma/ChromaVectorStoreAutoConfigurationIT.java
View File

@@ -19,7 +19,7 @@ import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.chromadb.ChromaDBContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
@@ -37,19 +37,19 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
* @author Eddú Meléndez
*/
@Testcontainers
public class ChromaVectorStoreAutoConfigurationIT {
@Container
static GenericContainer<?> chromaContainer = new GenericContainer<>("ghcr.io/chroma-core/chroma:0.4.15")
.withExposedPorts(8000);
static ChromaDBContainer chroma = new ChromaDBContainer("ghcr.io/chroma-core/chroma:0.4.15");
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(ChromaVectorStoreAutoConfiguration.class))
.withUserConfiguration(Config.class)
.withPropertyValues("spring.ai.vectorstore.chroma.client.host=http://" + chromaContainer.getHost(),
"spring.ai.vectorstore.chroma.client.port=" + chromaContainer.getMappedPort(8000),
.withPropertyValues("spring.ai.vectorstore.chroma.client.host=http://" + chroma.getHost(),
"spring.ai.vectorstore.chroma.client.port=" + chroma.getMappedPort(8000),
"spring.ai.vectorstore.chroma.store.collectionName=TestCollection");
@Test

41
spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/milvus/MilvusVectorStoreAutoConfigurationIT.java
View File

@@ -15,17 +15,11 @@
*/
package org.springframework.ai.autoconfigure.vectorstore.milvus;
import java.io.File;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.DockerComposeContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.springframework.ai.ResourceUtils;
@@ -38,46 +32,25 @@ import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.util.FileSystemUtils;
import org.testcontainers.milvus.MilvusContainer;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
* @author Eddú Meléndez
*/
@Testcontainers
public class MilvusVectorStoreAutoConfigurationIT {
private static DockerComposeContainer milvusContainer;
private static final File TEMP_FOLDER = new File("target/test-" + UUID.randomUUID().toString());
@Container
private static MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.3.8");
List<Document> documents = List.of(
new Document(ResourceUtils.getText("classpath:/test/data/spring.ai.txt"), Map.of("spring", "great")),
new Document(ResourceUtils.getText("classpath:/test/data/time.shelter.txt")), new Document(
ResourceUtils.getText("classpath:/test/data/great.depression.txt"), Map.of("depression", "bad")));
@BeforeAll
public static void beforeAll() {
FileSystemUtils.deleteRecursively(TEMP_FOLDER);
TEMP_FOLDER.mkdirs();
milvusContainer = new DockerComposeContainer(new File("src/test/resources/milvus/docker-compose.yml"))
.withEnv("DOCKER_VOLUME_DIRECTORY", TEMP_FOLDER.getAbsolutePath())
.withExposedService("standalone", 19530)
.withExposedService("standalone", 9091,
Wait.forHttp("/healthz").forPort(9091).forStatusCode(200).forStatusCode(401))
.waitingFor("standalone", Wait.forLogMessage(".*Proxy successfully started.*\\s", 1)
.withStartupTimeout(Duration.ofSeconds(100)));
milvusContainer.start();
}
@AfterAll
public static void afterAll() {
milvusContainer.stop();
FileSystemUtils.deleteRecursively(TEMP_FOLDER);
}
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(MilvusVectorStoreAutoConfiguration.class))
.withUserConfiguration(Config.class);
@@ -90,8 +63,8 @@ public class MilvusVectorStoreAutoConfigurationIT {
"spring.ai.vectorstore.milvus.embeddingDimension=384",
"spring.ai.vectorstore.milvus.collectionName=myTestCollection",
"spring.ai.vectorstore.milvus.client.host=" + milvusContainer.getServiceHost("standalone", 19530),
"spring.ai.vectorstore.milvus.client.port=" + milvusContainer.getServicePort("standalone", 19530))
"spring.ai.vectorstore.milvus.client.host=" + milvus.getHost(),
"spring.ai.vectorstore.milvus.client.port=" + milvus.getMappedPort(19530))
.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
vectorStore.add(documents);

13
spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/weaviate/WeaviateVectorStoreAutoConfigurationTests.java
View File

@@ -19,7 +19,6 @@ import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
@@ -33,29 +32,25 @@ import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.testcontainers.weaviate.WeaviateContainer;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
* @author Eddú Meléndez
*/
@Testcontainers
public class WeaviateVectorStoreAutoConfigurationTests {
@Container
static GenericContainer<?> weaviateContainer = new GenericContainer<>("semitechnologies/weaviate:1.22.4")
.withEnv("AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED", "true")
.withEnv("PERSISTENCE_DATA_PATH", "/var/lib/weaviate")
.withEnv("QUERY_DEFAULTS_LIMIT", "25")
.withEnv("DEFAULT_VECTORIZER_MODULE", "none")
.withEnv("CLUSTER_HOSTNAME", "node1")
.withExposedPorts(8080);
static WeaviateContainer weaviate = new WeaviateContainer("semitechnologies/weaviate:1.22.4");
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(WeaviateVectorStoreAutoConfiguration.class))
.withUserConfiguration(Config.class)
.withPropertyValues("spring.ai.vectorstore.weaviate.scheme=http",
"spring.ai.vectorstore.weaviate.host=localhost:" + weaviateContainer.getMappedPort(8080),
"spring.ai.vectorstore.weaviate.host=" + weaviate.getHttpHostAddress(),
"spring.ai.vectorstore.weaviate.filter-field.country=TEXT",
"spring.ai.vectorstore.weaviate.filter-field.year=NUMBER",
"spring.ai.vectorstore.weaviate.filter-field.active=BOOLEAN",

62
spring-ai-spring-boot-autoconfigure/src/test/resources/milvus/docker-compose.yml
View File

@@ -1,62 +0,0 @@
version: '3.5'
services:
etcd:
image: quay.io/coreos/etcd:v3.5.5
ports:
- "2379:2379"
environment:
- ETCD_AUTO_COMPACTION_MODE=revision
- ETCD_AUTO_COMPACTION_RETENTION=1000
- ETCD_QUOTA_BACKEND_BYTES=4294967296
- ETCD_SNAPSHOT_COUNT=50000
volumes:
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
healthcheck:
test: ["CMD", "etcdctl", "endpoint", "health"]
interval: 30s
timeout: 20s
retries: 3
minio:
image: minio/minio:RELEASE.2023-11-11T08-14-41Z
environment:
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
ports:
- "9001:9001"
- "9000:9000"
volumes:
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
command: minio server /minio_data --console-address ":9001"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
standalone:
image: milvusdb/milvus:v2.3.5
command: ["milvus", "run", "standalone"]
environment:
ETCD_ENDPOINTS: etcd:2379
MINIO_ADDRESS: minio:9000
volumes:
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
interval: 30s
start_period: 90s
timeout: 20s
retries: 3
ports:
- "19530:19530"
- "9091:9091"
depends_on:
- "etcd"
- "minio"
networks:
default:
name: milvus