Add support for Typesense Service Connection

Docker Compose and Testcontainers Service Connection support for
Typesense.
This commit is contained in:
Eddú Meléndez
2024-06-18 22:46:09 -05:00
committed by Christian Tzolov
parent 16c531c36c
commit 6ce998f9d3
17 changed files with 439 additions and 78 deletions

View File

@@ -43,6 +43,9 @@ The following service connection factories are provided in the `spring-ai-spring
| `RedisConnectionDetails`
| Containers named `redis/redis-stack-server`
| `TypesenseConnectionDetails`
| Containers named `typesense/typesense`
| `WeaviateConnectionDetails`
| Containers named `semitechnologies/weaviate`, `cr.weaviate.io/semitechnologies/weaviate`
|====

View File

@@ -46,6 +46,9 @@ The following service connection factories are provided in the `spring-ai-spring
| `RedisConnectionDetails`
| Containers of type `RedisStackContainer`
| `TypesenseConnectionDetails`
| Containers named "typesense/typesense"
| `WeaviateConnectionDetails`
| Containers of type `WeaviateContainer`
|====

View File

@@ -1,3 +1,18 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.autoconfigure.vectorstore.typesense;
import org.springframework.boot.autoconfigure.service.connection.ConnectionDetails;
@@ -11,6 +26,8 @@ public interface TypesenseConnectionDetails extends ConnectionDetails {
String getProtocol();
String getPort();
int getPort();
String getApiKey();
}

View File

@@ -14,7 +14,7 @@ public class TypesenseServiceClientProperties {
private String host = "localhost";
private String port = "8108";
private int port = 8108;
/**
* Typesense API key. This is the default api key when the user follows the Typesense
@@ -38,11 +38,11 @@ public class TypesenseServiceClientProperties {
this.host = host;
}
public String getPort() {
public int getPort() {
return port;
}
public void setPort(String port) {
public void setPort(int port) {
this.port = port;
}

View File

@@ -1,9 +1,23 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.autoconfigure.vectorstore.typesense;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.vectorstore.TypesenseVectorStore;
import org.springframework.ai.vectorstore.TypesenseVectorStore.TypesenseVectorStoreConfig;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -19,6 +33,7 @@ import java.util.List;
/**
* @author Pablo Sanchidrian Herrera
* @author Eddú Meléndez
*/
@AutoConfiguration
@ConditionalOnClass({ TypesenseVectorStore.class, EmbeddingModel.class })
@@ -47,16 +62,16 @@ public class TypesenseVectorStoreAutoConfiguration {
@Bean
@ConditionalOnMissingBean
public Client typesenseClient(TypesenseServiceClientProperties clientProperties,
TypesenseConnectionDetails connectionDetails) {
public Client typesenseClient(TypesenseConnectionDetails connectionDetails) {
List<Node> nodes = new ArrayList<>();
nodes.add(new Node(clientProperties.getProtocol(), clientProperties.getHost(), clientProperties.getPort()));
nodes.add(new Node(connectionDetails.getProtocol(), connectionDetails.getHost(),
String.valueOf(connectionDetails.getPort())));
Configuration configuration = new Configuration(nodes, Duration.ofSeconds(5), clientProperties.getApiKey());
Configuration configuration = new Configuration(nodes, Duration.ofSeconds(5), connectionDetails.getApiKey());
return new Client(configuration);
}
private static class PropertiesTypesenseConnectionDetails implements TypesenseConnectionDetails {
static class PropertiesTypesenseConnectionDetails implements TypesenseConnectionDetails {
private final TypesenseServiceClientProperties properties;
@@ -75,10 +90,15 @@ public class TypesenseVectorStoreAutoConfiguration {
}
@Override
public String getPort() {
public int getPort() {
return this.properties.getPort();
}
@Override
public String getApiKey() {
return this.properties.getApiKey();
}
}
}

View File

@@ -1,7 +1,20 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.autoconfigure.vectorstore.typesense;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.ai.ResourceUtils;
import org.springframework.ai.document.Document;
@@ -13,53 +26,34 @@ import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.util.FileSystemUtils;
import org.testcontainers.containers.BindMode;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import java.io.File;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Pablo Sanchidrian Herrera
* @author Eddú Meléndez
*/
@Testcontainers
public class TypesenseVectorStoreAutoConfigurationIT {
private static GenericContainer<?> typesenseContainer;
private static final File TEMP_FOLDER = new File("target/test-" + UUID.randomUUID().toString());
@Container
private static final GenericContainer<?> typesenseContainer = new GenericContainer<>("typesense/typesense:26.0")
.withExposedPorts(8108)
.withCommand("--data-dir", "/tmp", "--api-key=xyz", "--enable-cors")
.withStartupTimeout(Duration.ofSeconds(100));
List<Document> documents = List.of(
new Document(ResourceUtils.getText("classpath:/test/data/spring.ai.txt"), Map.of("spring", "great")),
new Document(ResourceUtils.getText("classpath:/test/data/time.shelter.txt")), new Document(
ResourceUtils.getText("classpath:/test/data/great.depression.txt"), Map.of("depression", "bad")));
@BeforeAll
public static void beforeAll() {
FileSystemUtils.deleteRecursively(TEMP_FOLDER);
TEMP_FOLDER.mkdirs();
typesenseContainer = new GenericContainer<>("typesense/typesense:26.0").withExposedPorts(8108)
.withCommand("--data-dir", "/data", "--api-key=xyz", "--enable-cors")
.withFileSystemBind(TEMP_FOLDER.getAbsolutePath(), "/data", BindMode.READ_WRITE)
.withStartupTimeout(Duration.ofSeconds(100));
typesenseContainer.start();
}
@AfterAll
public static void afterAll() {
typesenseContainer.stop();
FileSystemUtils.deleteRecursively(TEMP_FOLDER);
}
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(TypesenseVectorStoreAutoConfiguration.class))
.withUserConfiguration(Config.class);

View File

@@ -0,0 +1,82 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.docker.compose.service.connection.typesense;
import org.springframework.ai.autoconfigure.vectorstore.typesense.TypesenseConnectionDetails;
import org.springframework.boot.docker.compose.core.RunningService;
import org.springframework.boot.docker.compose.service.connection.DockerComposeConnectionDetailsFactory;
import org.springframework.boot.docker.compose.service.connection.DockerComposeConnectionSource;
/**
* @author Eddú Meléndez
*/
public class TypesenseDockerComposeConnectionDetailsFactory
extends DockerComposeConnectionDetailsFactory<TypesenseConnectionDetails> {
private static final int TYPESENSE_PORT = 8108;
protected TypesenseDockerComposeConnectionDetailsFactory() {
super("typesense/typesense");
}
@Override
protected TypesenseConnectionDetails getDockerComposeConnectionDetails(DockerComposeConnectionSource source) {
return new TypesenseComposeConnectionDetails(source.getRunningService());
}
/**
* {@link TypesenseConnectionDetails} backed by a {@code Typesense}
* {@link RunningService}.
*/
static class TypesenseComposeConnectionDetails extends DockerComposeConnectionDetails
implements TypesenseConnectionDetails {
private final TypesenseEnvironment environment;
private final String host;
private final int port;
TypesenseComposeConnectionDetails(RunningService service) {
super(service);
this.environment = new TypesenseEnvironment(service.env());
this.host = service.host();
this.port = service.ports().get(TYPESENSE_PORT);
}
@Override
public String getHost() {
return this.host;
}
@Override
public String getProtocol() {
return "http";
}
@Override
public int getPort() {
return this.port;
}
@Override
public String getApiKey() {
return this.environment.getApiKey();
}
}
}

View File

@@ -0,0 +1,17 @@
package org.springframework.ai.docker.compose.service.connection.typesense;
import java.util.Map;
class TypesenseEnvironment {
private final String apiKey;
TypesenseEnvironment(Map<String, String> env) {
this.apiKey = env.get("TYPESENSE_API_KEY");
}
public String getApiKey() {
return this.apiKey;
}
}

View File

@@ -3,4 +3,5 @@ org.springframework.ai.docker.compose.service.connection.chroma.ChromaDockerComp
org.springframework.ai.docker.compose.service.connection.ollama.OllamaDockerComposeConnectionDetailsFactory,\
org.springframework.ai.docker.compose.service.connection.qdrant.QdrantDockerComposeConnectionDetailsFactory,\
org.springframework.ai.docker.compose.service.connection.redis.RedisDockerComposeConnectionDetailsFactory,\
org.springframework.ai.docker.compose.service.connection.typesense.TypesenseDockerComposeConnectionDetailsFactory,\
org.springframework.ai.docker.compose.service.connection.weaviate.WeaviateDockerComposeConnectionDetailsFactory

View File

@@ -0,0 +1,40 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.docker.compose.service.connection.typesense;
import org.junit.jupiter.api.Test;
import org.springframework.ai.autoconfigure.vectorstore.typesense.TypesenseConnectionDetails;
import org.springframework.boot.docker.compose.service.connection.test.AbstractDockerComposeIntegrationTests;
import org.testcontainers.utility.DockerImageName;
import static org.assertj.core.api.Assertions.assertThat;
class TypesenseDockerComposeConnectionDetailsFactoryTests extends AbstractDockerComposeIntegrationTests {
TypesenseDockerComposeConnectionDetailsFactoryTests() {
super("typesense-compose.yaml", DockerImageName.parse("typesense/typesense:26.0"));
}
@Test
void runCreatesConnectionDetails() {
TypesenseConnectionDetails connectionDetails = run(TypesenseConnectionDetails.class);
assertThat(connectionDetails.getHost()).isNotNull();
assertThat(connectionDetails.getPort()).isGreaterThan(0);
assertThat(connectionDetails.getProtocol()).isEqualTo("http");
assertThat(connectionDetails.getApiKey()).isEqualTo("secret");
}
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.docker.compose.service.connection.typesense;
import org.junit.jupiter.api.Test;
import java.util.Collections;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
class TypesenseEnvironmentTests {
@Test
void getApiKeyWhenNoApiKey() {
TypesenseEnvironment environment = new TypesenseEnvironment(Collections.emptyMap());
assertThat(environment.getApiKey()).isNull();
}
@Test
void getApiKeyWhenHasApiKey() {
TypesenseEnvironment environment = new TypesenseEnvironment(Map.of("TYPESENSE_API_KEY", "secret"));
assertThat(environment.getApiKey()).isEqualTo("secret");
}
}

View File

@@ -0,0 +1,8 @@
services:
typesense:
image: '{imageName}'
ports:
- '8108'
command: '--data-dir /tmp --enable-cors'
environment:
- TYPESENSE_API_KEY=secret

View File

@@ -115,6 +115,14 @@
<optional>true</optional>
</dependency>
<!-- Typesense Vector Store-->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-typesense-store</artifactId>
<version>${project.parent.version}</version>
<optional>true</optional>
</dependency>
<!-- test dependencies -->
<dependency>

View File

@@ -0,0 +1,70 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.testcontainers.service.connection.typesense;
import org.springframework.ai.autoconfigure.vectorstore.typesense.TypesenseConnectionDetails;
import org.springframework.boot.testcontainers.service.connection.ContainerConnectionDetailsFactory;
import org.springframework.boot.testcontainers.service.connection.ContainerConnectionSource;
import org.testcontainers.containers.Container;
/**
* @author Eddú Meléndez
*/
class TypesenseContainerConnectionDetailsFactory
extends ContainerConnectionDetailsFactory<Container<?>, TypesenseConnectionDetails> {
TypesenseContainerConnectionDetailsFactory() {
super("typesense/typesense");
}
@Override
protected TypesenseConnectionDetails getContainerConnectionDetails(ContainerConnectionSource<Container<?>> source) {
return new TypesenseContainerConnectionDetails(source);
}
/**
* {@link TypesenseConnectionDetails} backed by a {@link ContainerConnectionSource}.
*/
private static final class TypesenseContainerConnectionDetails extends ContainerConnectionDetails<Container<?>>
implements TypesenseConnectionDetails {
private TypesenseContainerConnectionDetails(ContainerConnectionSource<Container<?>> source) {
super(source);
}
@Override
public String getHost() {
return getContainer().getHost();
}
@Override
public String getProtocol() {
return "http";
}
@Override
public int getPort() {
return getContainer().getMappedPort(8108);
}
@Override
public String getApiKey() {
return getContainer().getEnvMap().get("TYPESENSE_API_KEY");
}
}
}

View File

@@ -4,4 +4,5 @@ org.springframework.ai.testcontainers.service.connection.milvus.MilvusContainerC
org.springframework.ai.testcontainers.service.connection.ollama.OllamaContainerConnectionDetailsFactory,\
org.springframework.ai.testcontainers.service.connection.qdrant.QdrantContainerConnectionDetailsFactory,\
org.springframework.ai.testcontainers.service.connection.redis.RedisContainerConnectionDetailsFactory,\
org.springframework.ai.testcontainers.service.connection.typesense.TypesenseContainerConnectionDetailsFactory,\
org.springframework.ai.testcontainers.service.connection.weaviate.WeaviateContainerConnectionDetailsFactory

View File

@@ -0,0 +1,84 @@
package org.springframework.ai.testcontainers.service.connection.typesense;
import org.junit.jupiter.api.Test;
import org.springframework.ai.ResourceUtils;
import org.springframework.ai.autoconfigure.vectorstore.typesense.TypesenseVectorStoreAutoConfiguration;
import org.springframework.ai.document.Document;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.transformers.TransformersEmbeddingModel;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@SpringJUnitConfig
@TestPropertySource(properties = { "spring.ai.vectorstore.typesense.embeddingDimension=384",
"spring.ai.vectorstore.typesense.collectionName=myTestCollection" })
@Testcontainers
class TypesenseContainerConnectionDetailsFactoryTest {
@Container
@ServiceConnection
private static final GenericContainer<?> typesense = new GenericContainer<>("typesense/typesense:26.0")
.withExposedPorts(8108)
.withCommand("--data-dir", "/tmp", "--enable-cors")
.withEnv("TYPESENSE_API_KEY", "secret")
.withStartupTimeout(Duration.ofSeconds(100));
List<Document> documents = List.of(
new Document(ResourceUtils.getText("classpath:/test/data/spring.ai.txt"), Map.of("spring", "great")),
new Document(ResourceUtils.getText("classpath:/test/data/time.shelter.txt")), new Document(
ResourceUtils.getText("classpath:/test/data/great.depression.txt"), Map.of("depression", "bad")));
@Autowired
private VectorStore vectorStore;
@Test
public void addAndSearch() {
this.vectorStore.add(documents);
List<Document> results = this.vectorStore.similaritySearch(SearchRequest.query("Spring").withTopK(1));
assertThat(results).hasSize(1);
Document resultDoc = results.get(0);
assertThat(resultDoc.getId()).isEqualTo(documents.get(0).getId());
assertThat(resultDoc.getContent())
.contains("Spring AI provides abstractions that serve as the foundation for developing AI applications.");
assertThat(resultDoc.getMetadata()).hasSize(2);
assertThat(resultDoc.getMetadata()).containsKeys("spring", "distance");
this.vectorStore.delete(documents.stream().map(doc -> doc.getId()).toList());
results = this.vectorStore.similaritySearch(SearchRequest.query("Spring").withTopK(1));
assertThat(results).hasSize(0);
}
@Configuration(proxyBeanMethods = false)
@ImportAutoConfiguration(TypesenseVectorStoreAutoConfiguration.class)
static class Config {
@Bean
public EmbeddingModel embeddingModel() {
return new TransformersEmbeddingModel();
}
}
}

View File

@@ -1,19 +1,5 @@
package org.springframework.ai.vectorstore;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.springframework.ai.document.Document;
import org.springframework.ai.embedding.EmbeddingModel;
@@ -25,7 +11,6 @@ import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.DefaultResourceLoader;
import org.testcontainers.containers.BindMode;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
@@ -33,28 +18,28 @@ import org.typesense.api.Client;
import org.typesense.api.Configuration;
import org.typesense.resources.Node;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Pablo Sanchidrian Herrera
* @author Eddú Meléndez
*/
@Testcontainers
public class TypesenseVectorStoreIT {
private static Path tempDirectory;
static {
try {
tempDirectory = Files.createTempDirectory("typesense-test");
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
@Container
private static GenericContainer<?> typesenseContainer = new GenericContainer<>("typesense/typesense:26.0")
.withExposedPorts(8108)
.withCommand("--data-dir", "/data", "--api-key=xyz", "--enable-cors")
.withFileSystemBind(tempDirectory.toString(), "/data", BindMode.READ_WRITE);
.withCommand("--data-dir", "/tmp", "--api-key=xyz", "--enable-cors");
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(TestApplication.class);
@@ -271,15 +256,4 @@ public class TypesenseVectorStoreIT {
}
@AfterAll
static void deleteContainer() {
if (typesenseContainer != null) {
typesenseContainer.stop();
}
if (tempDirectory != null) {
tempDirectory.toFile().delete();
}
}
}