diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java index 5d1adf5bd..4efbc3a77 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java @@ -15,12 +15,12 @@ */ package org.springframework.ai.ollama; +import java.util.Base64; import java.util.List; import reactor.core.publisher.Flux; import org.springframework.ai.chat.ChatClient; -import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.ChatResponse; import org.springframework.ai.chat.Generation; import org.springframework.ai.chat.StreamingChatClient; @@ -28,11 +28,14 @@ import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.MessageType; import org.springframework.ai.chat.metadata.ChatGenerationMetadata; import org.springframework.ai.chat.metadata.Usage; +import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.model.ModelOptionsUtils; import org.springframework.ai.ollama.api.OllamaApi; import org.springframework.ai.ollama.api.OllamaApi.Message.Role; import org.springframework.ai.ollama.api.OllamaOptions; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; /** @@ -59,10 +62,17 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient { /** * Default options to be used for all chat requests. */ - private OllamaOptions defaultOptions = OllamaOptions.create().withModel(OllamaOptions.DEFAULT_MODEL); + private OllamaOptions defaultOptions; public OllamaChatClient(OllamaApi chatApi) { + this(chatApi, OllamaOptions.create().withModel(OllamaOptions.DEFAULT_MODEL)); + } + + public OllamaChatClient(OllamaApi chatApi, OllamaOptions defaultOptions) { + Assert.notNull(chatApi, "OllamaApi must not be null"); + Assert.notNull(defaultOptions, "DefaultOptions must not be null"); this.chatApi = chatApi; + this.defaultOptions = defaultOptions; } /** @@ -74,6 +84,9 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient { return this; } + /** + * @deprecated Use {@link OllamaOptions} constructor instead. + */ public OllamaChatClient withDefaultOptions(OllamaOptions options) { this.defaultOptions = options; return this; @@ -83,6 +96,7 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient { public ChatResponse call(Prompt prompt) { OllamaApi.ChatResponse response = this.chatApi.chat(ollamaChatRequest(prompt, false)); + var generator = new Generation(response.message().content()); if (response.promptEvalCount() != null && response.evalCount() != null) { generator = generator @@ -132,7 +146,15 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient { .filter(message -> message.getMessageType() == MessageType.USER || message.getMessageType() == MessageType.ASSISTANT || message.getMessageType() == MessageType.SYSTEM) - .map(m -> OllamaApi.Message.builder(toRole(m)).withContent(m.getContent()).build()) + .map(m -> { + var messageBuilder = OllamaApi.Message.builder(toRole(m)).withContent(m.getContent()); + + if (!CollectionUtils.isEmpty(m.getMedia())) { + messageBuilder + .withImages(m.getMedia().stream().map(media -> this.fromMediaData(media.getData())).toList()); + } + return messageBuilder.build(); + }) .toList(); // runtime options @@ -163,6 +185,19 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient { .build(); } + private String fromMediaData(Object mediaData) { + if (mediaData instanceof byte[] bytes) { + return Base64.getEncoder().encodeToString(bytes); + } + else if (mediaData instanceof String text) { + return text; + } + else { + throw new IllegalArgumentException("Unsupported media data type: " + mediaData.getClass().getSimpleName()); + } + + } + private OllamaApi.Message.Role toRole(Message message) { switch (message.getMessageType()) { diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java index 9b5488fbb..18f2e1891 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java @@ -333,13 +333,14 @@ public class OllamaApi { * * @param role The role of the message of type {@link Role}. * @param content The content of the message. - * @param images The list of images to send with the message. + * @param images The list of base64-encoded images to send with the message. + * Requires multimodal models such as llava or bakllava. */ @JsonInclude(Include.NON_NULL) public record Message( @JsonProperty("role") Role role, @JsonProperty("content") String content, - @JsonProperty("images") List images) { + @JsonProperty("images") List images) { /** * The role of the message in the conversation. @@ -369,7 +370,7 @@ public class OllamaApi { private final Role role; private String content; - private List images; + private List images; public Builder(Role role) { this.role = role; @@ -380,7 +381,7 @@ public class OllamaApi { return this; } - public Builder withImages(List images) { + public Builder withImages(List images) { this.images = images; return this; } diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java index 72941eff8..b76cae1dd 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java @@ -62,7 +62,7 @@ class OllamaChatClientIT { private static final Log logger = LogFactory.getLog(OllamaChatClientIT.class); @Container - static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434); + static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434); static String baseUrl; diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientMultimodalIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientMultimodalIT.java new file mode 100644 index 000000000..bc1919a01 --- /dev/null +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientMultimodalIT.java @@ -0,0 +1,100 @@ +/* + * Copyright 2023 - 2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.ai.ollama; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import org.springframework.ai.chat.ChatResponse; +import org.springframework.ai.chat.messages.Media; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.ollama.api.OllamaApi; +import org.springframework.ai.ollama.api.OllamaOptions; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.util.MimeTypeUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest +@Testcontainers +@Disabled("For manual smoke testing only.") +class OllamaChatClientMultimodalIT { + + private static String MODEL = "llava"; + + private static final Log logger = LogFactory.getLog(OllamaChatClientIT.class); + + @Container + static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434); + + static String baseUrl; + + @BeforeAll + public static void beforeAll() throws IOException, InterruptedException { + logger.info("Start pulling the '" + MODEL + " ' generative ... would take several minutes ..."); + ollamaContainer.execInContainer("ollama", "pull", MODEL); + logger.info(MODEL + " pulling competed!"); + + baseUrl = "http://" + ollamaContainer.getHost() + ":" + ollamaContainer.getMappedPort(11434); + } + + @Autowired + private OllamaChatClient client; + + @Test + void multiModalityTest() throws IOException { + + byte[] imageData = new ClassPathResource("/test.png").getContentAsByteArray(); + + var userMessage = new UserMessage("Explain what do you see on this picture?", + List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData))); + + ChatResponse response = client.call(new Prompt(List.of(userMessage))); + + logger.info(response.getResult().getOutput().getContent()); + assertThat(response.getResult().getOutput().getContent()).contains("bananas", "apple", "basket"); + } + + @SpringBootConfiguration + public static class TestConfiguration { + + @Bean + public OllamaApi ollamaApi() { + return new OllamaApi(baseUrl); + } + + @Bean + public OllamaChatClient ollamaChat(OllamaApi ollamaApi) { + return new OllamaChatClient(ollamaApi, OllamaOptions.create().withModel(MODEL).withTemperature(0.9f)); + } + + } + +} \ No newline at end of file diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java index a4fa85b62..077c1d36d 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java @@ -45,7 +45,7 @@ class OllamaEmbeddingClientIT { private static final Log logger = LogFactory.getLog(OllamaApiIT.class); @Container - static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434); + static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434); static String baseUrl; diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java index 48319596c..339b69e66 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java @@ -50,7 +50,7 @@ public class OllamaApiIT { private static final Log logger = LogFactory.getLog(OllamaApiIT.class); @Container - static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434); + static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434); static OllamaApi ollamaApi; diff --git a/models/spring-ai-ollama/src/test/resources/test.png b/models/spring-ai-ollama/src/test/resources/test.png new file mode 100644 index 000000000..8abb4c81a Binary files /dev/null and b/models/spring-ai-ollama/src/test/resources/test.png differ diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc index 6a1263133..edca8ea97 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc @@ -144,7 +144,7 @@ Below is a simple code example extracted from https://github.com/spring-projects ---- byte[] imageData = new ClassPathResource("/multimodal.test.png").getContentAsByteArray(); -var userMessage = new UserMessage("Explain what do you see o this picture?", +var userMessage = new UserMessage("Explain what do you see on this picture?", List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData))); ChatResponse response = chatClient.call(new Prompt(List.of(userMessage))); diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc index a3fbcf784..7b796d647 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc @@ -123,6 +123,48 @@ ChatResponse response = chatClient.call( TIP: In addition to the model specific link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java[OllamaOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptions.java[ChatOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptionsBuilder.java[ChatOptionsBuilder#builder()]. + +== Multimodal + +Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. + +Presently, the https://ollama.com/library/llava[LLaVa] and https://ollama.com/library/bakllava[bakllava] Ollama models offer multimodal support. +For further details, refer to the link:https://llava-vl.github.io/[LLaVA: Large Language and Vision Assistant]. + +The Ollama link:https://github.com/ollama/ollama/blob/main/docs/api.md#parameters-1[Message API] provides an "images" parameter to incorporate a list of base64-encoded images with the message. + +Spring AI’s link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java[Message] interface facilitates multimodal AI models by introducing the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java[Media] type. +This type encompasses data and details regarding media attachments in messages, utilizing Spring’s `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data. + +Below is a straightforward code example excerpted from link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientMultimodalIT.java[OllamaChatClientMultimodalIT.java], illustrating the fusion of user text with an image. + +[source,java] +---- +byte[] imageData = new ClassPathResource("/multimodal.test.png").getContentAsByteArray(); + +var userMessage = new UserMessage("Explain what do you see on this picture?", + List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData))); + +ChatResponse response = chatClient.call( + new Prompt(List.of(userMessage), OllamaOptions.create().withModel("llava"))); + +logger.info(response.getResult().getOutput().getContent()); +---- + +It takes as an input the `multimodal.test.png` image: + +image::multimodal.test.png[Multimodal Test Image, 200, 200, align="left"] + +along with the text message "Explain what do you see on this picture?", and generates a response like this: + +---- +The image shows a small metal basket filled with ripe bananas and red apples. The basket is placed on a surface, +which appears to be a table or countertop, as there's a hint of what seems like a kitchen cabinet or drawer in +the background. There's also a gold-colored ring visible behind the basket, which could indicate that this +photo was taken in an area with metallic decorations or fixtures. The overall setting suggests a home environment +where fruits are being displayed, possibly for convenience or aesthetic purposes. +---- + == Sample Controller https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-openai-spring-boot-starter` to your pom (or gradle) dependencies. diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java index 58d211260..8e46f7477 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java @@ -48,7 +48,7 @@ public class OllamaEmbeddingAutoConfigurationIT { private static String MODEL_NAME = "orca-mini"; @Container - static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434); + static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434); static String baseUrl; diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java index 16ef622d3..3fe056c72 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java @@ -17,6 +17,6 @@ package org.springframework.ai.autoconfigure.ollama; public class OllamaImage { - static final String IMAGE = "ollama/ollama:0.1.23"; + static final String IMAGE = "ollama/ollama:0.1.29"; } diff --git a/spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaContainerConnectionDetailsFactoryTest.java b/spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaContainerConnectionDetailsFactoryTest.java index 5b79e662f..aa470135a 100644 --- a/spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaContainerConnectionDetailsFactoryTest.java +++ b/spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaContainerConnectionDetailsFactoryTest.java @@ -53,7 +53,7 @@ class OllamaContainerConnectionDetailsFactoryTest { @Container @ServiceConnection - static OllamaContainer ollama = new OllamaContainer("ollama/ollama:0.1.23"); + static OllamaContainer ollama = new OllamaContainer("ollama/ollama:0.1.29"); @Autowired private OllamaEmbeddingClient embeddingClient;