Add Ollama multimodality support

- Requires LLaVa or Bakllava models.
  - Extend the Ollama chat client to support the Spring AI Medi type inputs.
  - Add OllamaChatClientMultimodalIT.java
  - Add Multimodal section in Ollama's docs.

 Resolves #421

models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java

@@ -15,12 +15,12 @@
  */
 package org.springframework.ai.ollama;
 
+import java.util.Base64;
 import java.util.List;
 
 import reactor.core.publisher.Flux;
 
 import org.springframework.ai.chat.ChatClient;
-import org.springframework.ai.chat.prompt.ChatOptions;
 import org.springframework.ai.chat.ChatResponse;
 import org.springframework.ai.chat.Generation;
 import org.springframework.ai.chat.StreamingChatClient;
@@ -28,11 +28,14 @@ import org.springframework.ai.chat.messages.Message;
 import org.springframework.ai.chat.messages.MessageType;
 import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
 import org.springframework.ai.chat.metadata.Usage;
+import org.springframework.ai.chat.prompt.ChatOptions;
 import org.springframework.ai.chat.prompt.Prompt;
 import org.springframework.ai.model.ModelOptionsUtils;
 import org.springframework.ai.ollama.api.OllamaApi;
 import org.springframework.ai.ollama.api.OllamaApi.Message.Role;
 import org.springframework.ai.ollama.api.OllamaOptions;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 
 /**
@@ -59,10 +62,17 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient {
 	/**
 	 * Default options to be used for all chat requests.
 	 */
-	private OllamaOptions defaultOptions = OllamaOptions.create().withModel(OllamaOptions.DEFAULT_MODEL);
+	private OllamaOptions defaultOptions;
 
 	public OllamaChatClient(OllamaApi chatApi) {
+		this(chatApi, OllamaOptions.create().withModel(OllamaOptions.DEFAULT_MODEL));
+	}
+
+	public OllamaChatClient(OllamaApi chatApi, OllamaOptions defaultOptions) {
+		Assert.notNull(chatApi, "OllamaApi must not be null");
+		Assert.notNull(defaultOptions, "DefaultOptions must not be null");
 		this.chatApi = chatApi;
+		this.defaultOptions = defaultOptions;
 	}
 
 	/**
@@ -74,6 +84,9 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient {
 		return this;
 	}
 
+	/**
+	 * @deprecated Use {@link OllamaOptions} constructor instead.
+	 */
 	public OllamaChatClient withDefaultOptions(OllamaOptions options) {
 		this.defaultOptions = options;
 		return this;
@@ -83,6 +96,7 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient {
 	public ChatResponse call(Prompt prompt) {
 
 		OllamaApi.ChatResponse response = this.chatApi.chat(ollamaChatRequest(prompt, false));
+
 		var generator = new Generation(response.message().content());
 		if (response.promptEvalCount() != null && response.evalCount() != null) {
 			generator = generator
@@ -132,7 +146,15 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient {
 			.filter(message -> message.getMessageType() == MessageType.USER
 					|| message.getMessageType() == MessageType.ASSISTANT
 					|| message.getMessageType() == MessageType.SYSTEM)
-			.map(m -> OllamaApi.Message.builder(toRole(m)).withContent(m.getContent()).build())
+			.map(m -> {
+				var messageBuilder = OllamaApi.Message.builder(toRole(m)).withContent(m.getContent());
+
+				if (!CollectionUtils.isEmpty(m.getMedia())) {
+					messageBuilder
+						.withImages(m.getMedia().stream().map(media -> this.fromMediaData(media.getData())).toList());
+				}
+				return messageBuilder.build();
+			})
 			.toList();
 
 		// runtime options
@@ -163,6 +185,19 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient {
 			.build();
 	}
 
+	private String fromMediaData(Object mediaData) {
+		if (mediaData instanceof byte[] bytes) {
+			return Base64.getEncoder().encodeToString(bytes);
+		}
+		else if (mediaData instanceof String text) {
+			return text;
+		}
+		else {
+			throw new IllegalArgumentException("Unsupported media data type: " + mediaData.getClass().getSimpleName());
+		}
+
+	}
+
 	private OllamaApi.Message.Role toRole(Message message) {
 
 		switch (message.getMessageType()) {

models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java

@@ -333,13 +333,14 @@ public class OllamaApi {
 	 *
 	 * @param role The role of the message of type {@link Role}.
 	 * @param content The content of the message.
-	 * @param images The list of images to send with the message.
+	 * @param images The list of base64-encoded images to send with the message.
+	 * 				 Requires multimodal models such as llava or bakllava.
 	 */
 	@JsonInclude(Include.NON_NULL)
 	public record Message(
 			@JsonProperty("role") Role role,
 			@JsonProperty("content") String content,
-			@JsonProperty("images") List<byte[]> images) {
+			@JsonProperty("images") List<String> images) {
 
 		/**
 		 * The role of the message in the conversation.
@@ -369,7 +370,7 @@ public class OllamaApi {
 
 			private final Role role;
 			private String content;
-			private List<byte[]> images;
+			private List<String> images;
 
 			public Builder(Role role) {
 				this.role = role;
@@ -380,7 +381,7 @@ public class OllamaApi {
 				return this;
 			}
 
-			public Builder withImages(List<byte[]> images) {
+			public Builder withImages(List<String> images) {
 				this.images = images;
 				return this;
 			}

models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java

@@ -62,7 +62,7 @@ class OllamaChatClientIT {
 	private static final Log logger = LogFactory.getLog(OllamaChatClientIT.class);
 
 	@Container
-	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
+	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434);
 
 	static String baseUrl;
 

models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientMultimodalIT.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2023 - 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.ai.ollama;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import org.springframework.ai.chat.ChatResponse;
+import org.springframework.ai.chat.messages.Media;
+import org.springframework.ai.chat.messages.UserMessage;
+import org.springframework.ai.chat.prompt.Prompt;
+import org.springframework.ai.ollama.api.OllamaApi;
+import org.springframework.ai.ollama.api.OllamaOptions;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringBootConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.util.MimeTypeUtils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@SpringBootTest
+@Testcontainers
+@Disabled("For manual smoke testing only.")
+class OllamaChatClientMultimodalIT {
+
+	private static String MODEL = "llava";
+
+	private static final Log logger = LogFactory.getLog(OllamaChatClientIT.class);
+
+	@Container
+	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434);
+
+	static String baseUrl;
+
+	@BeforeAll
+	public static void beforeAll() throws IOException, InterruptedException {
+		logger.info("Start pulling the '" + MODEL + " ' generative ... would take several minutes ...");
+		ollamaContainer.execInContainer("ollama", "pull", MODEL);
+		logger.info(MODEL + " pulling competed!");
+
+		baseUrl = "http://" + ollamaContainer.getHost() + ":" + ollamaContainer.getMappedPort(11434);
+	}
+
+	@Autowired
+	private OllamaChatClient client;
+
+	@Test
+	void multiModalityTest() throws IOException {
+
+		byte[] imageData = new ClassPathResource("/test.png").getContentAsByteArray();
+
+		var userMessage = new UserMessage("Explain what do you see on this picture?",
+				List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData)));
+
+		ChatResponse response = client.call(new Prompt(List.of(userMessage)));
+
+		logger.info(response.getResult().getOutput().getContent());
+		assertThat(response.getResult().getOutput().getContent()).contains("bananas", "apple", "basket");
+	}
+
+	@SpringBootConfiguration
+	public static class TestConfiguration {
+
+		@Bean
+		public OllamaApi ollamaApi() {
+			return new OllamaApi(baseUrl);
+		}
+
+		@Bean
+		public OllamaChatClient ollamaChat(OllamaApi ollamaApi) {
+			return new OllamaChatClient(ollamaApi, OllamaOptions.create().withModel(MODEL).withTemperature(0.9f));
+		}
+
+	}
+
+}

models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java

@@ -45,7 +45,7 @@ class OllamaEmbeddingClientIT {
 	private static final Log logger = LogFactory.getLog(OllamaApiIT.class);
 
 	@Container
-	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
+	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434);
 
 	static String baseUrl;
 

models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java

@@ -50,7 +50,7 @@ public class OllamaApiIT {
 	private static final Log logger = LogFactory.getLog(OllamaApiIT.class);
 
 	@Container
-	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
+	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434);
 
 	static OllamaApi ollamaApi;
 

spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc

@@ -144,7 +144,7 @@ Below is a simple code example extracted from https://github.com/spring-projects
 ----
 byte[] imageData = new ClassPathResource("/multimodal.test.png").getContentAsByteArray();
 
-var userMessage = new UserMessage("Explain what do you see o this picture?",
+var userMessage = new UserMessage("Explain what do you see on this picture?",
         List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData)));
 
 ChatResponse response = chatClient.call(new Prompt(List.of(userMessage)));

spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc

@@ -123,6 +123,48 @@ ChatResponse response = chatClient.call(
 
 TIP: In addition to the model specific link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java[OllamaOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptions.java[ChatOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptionsBuilder.java[ChatOptionsBuilder#builder()].
 
+
+== Multimodal
+
+Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats.
+
+Presently, the https://ollama.com/library/llava[LLaVa] and https://ollama.com/library/bakllava[bakllava] Ollama models offer multimodal support.
+For further details, refer to the link:https://llava-vl.github.io/[LLaVA: Large Language and Vision Assistant].
+
+The Ollama link:https://github.com/ollama/ollama/blob/main/docs/api.md#parameters-1[Message API] provides an "images" parameter to incorporate a list of base64-encoded images with the message.
+
+Spring AI’s link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java[Message] interface facilitates multimodal AI models by introducing the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java[Media] type.
+This type encompasses data and details regarding media attachments in messages, utilizing Spring’s `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data.
+
+Below is a straightforward code example excerpted from link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientMultimodalIT.java[OllamaChatClientMultimodalIT.java], illustrating the fusion of user text with an image.
+
+[source,java]
+----
+byte[] imageData = new ClassPathResource("/multimodal.test.png").getContentAsByteArray();
+
+var userMessage = new UserMessage("Explain what do you see on this picture?",
+        List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData)));
+
+ChatResponse response = chatClient.call(
+    new Prompt(List.of(userMessage), OllamaOptions.create().withModel("llava")));
+
+logger.info(response.getResult().getOutput().getContent());
+----
+
+It takes as an input the `multimodal.test.png` image:
+
+image::multimodal.test.png[Multimodal Test Image, 200, 200, align="left"]
+
+along with the text message "Explain what do you see on this picture?", and generates a response like this:
+
+----
+The image shows a small metal basket filled with ripe bananas and red apples. The basket is placed on a surface,
+which appears to be a table or countertop, as there's a hint of what seems like a kitchen cabinet or drawer in
+the background. There's also a gold-colored ring visible behind the basket, which could indicate that this
+photo was taken in an area with metallic decorations or fixtures. The overall setting suggests a home environment
+where fruits are being displayed, possibly for convenience or aesthetic purposes.
+----
+
 == Sample Controller
 
 https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-openai-spring-boot-starter` to your pom (or gradle) dependencies.

spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java

@@ -48,7 +48,7 @@ public class OllamaEmbeddingAutoConfigurationIT {
 	private static String MODEL_NAME = "orca-mini";
 
 	@Container
-	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
+	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.29").withExposedPorts(11434);
 
 	static String baseUrl;
 

spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java

@@ -17,6 +17,6 @@ package org.springframework.ai.autoconfigure.ollama;
 
 public class OllamaImage {
 
-	static final String IMAGE = "ollama/ollama:0.1.23";
+	static final String IMAGE = "ollama/ollama:0.1.29";
 
 }

spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaContainerConnectionDetailsFactoryTest.java

@@ -53,7 +53,7 @@ class OllamaContainerConnectionDetailsFactoryTest {
 
 	@Container
 	@ServiceConnection
-	static OllamaContainer ollama = new OllamaContainer("ollama/ollama:0.1.23");
+	static OllamaContainer ollama = new OllamaContainer("ollama/ollama:0.1.29");
 
 	@Autowired
 	private OllamaEmbeddingClient embeddingClient;