From f2820cd8af10eb70da6c06e53701e55abef229d7 Mon Sep 17 00:00:00 2001 From: Thomas Vitale Date: Wed, 18 Dec 2024 12:32:07 +0100 Subject: [PATCH] Mistral: Support Vision Modality Introduce multimodality support for Mistral AI, which currently supports text and vision modalities. Added integration tests and documentation for the new capability. Signed-off-by: Thomas Vitale --- .../ai/mistralai/MistralAiChatModel.java | 40 ++++++++- .../ai/mistralai/api/MistralAiApi.java | 79 +++++++++++++++++- .../ai/mistralai/MistralAiChatModelIT.java | 82 +++++++++++++++++-- .../ROOT/pages/api/chat/mistralai-chat.adoc | 56 +++++++++++++ .../modules/ROOT/pages/api/multimodality.adoc | 7 +- 5 files changed, 246 insertions(+), 18 deletions(-) diff --git a/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java b/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java index 9c1fde161..5871390d6 100644 --- a/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java +++ b/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java @@ -16,6 +16,8 @@ package org.springframework.ai.mistralai; +import java.util.ArrayList; +import java.util.Base64; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -27,6 +29,8 @@ import io.micrometer.observation.ObservationRegistry; import io.micrometer.observation.contextpropagation.ObservationThreadLocalAccessor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.ai.model.Media; +import org.springframework.util.MimeType; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -353,8 +357,19 @@ public class MistralAiChatModel extends AbstractToolCallSupport implements ChatM List chatCompletionMessages = prompt.getInstructions().stream().map(message -> { if (message instanceof UserMessage userMessage) { - return List.of(new MistralAiApi.ChatCompletionMessage(userMessage.getText(), - MistralAiApi.ChatCompletionMessage.Role.USER)); + Object content = message.getText(); + + if (!CollectionUtils.isEmpty(userMessage.getMedia())) { + List contentList = new ArrayList<>( + List.of(new ChatCompletionMessage.MediaContent(message.getText()))); + + contentList.addAll(userMessage.getMedia().stream().map(this::mapToMediaContent).toList()); + + content = contentList; + } + + return List + .of(new MistralAiApi.ChatCompletionMessage(content, MistralAiApi.ChatCompletionMessage.Role.USER)); } else if (message instanceof SystemMessage systemMessage) { return List.of(new MistralAiApi.ChatCompletionMessage(systemMessage.getText(), @@ -424,6 +439,27 @@ public class MistralAiChatModel extends AbstractToolCallSupport implements ChatM return request; } + private ChatCompletionMessage.MediaContent mapToMediaContent(Media media) { + return new ChatCompletionMessage.MediaContent(new ChatCompletionMessage.MediaContent.ImageUrl( + this.fromMediaData(media.getMimeType(), media.getData()))); + } + + private String fromMediaData(MimeType mimeType, Object mediaContentData) { + if (mediaContentData instanceof byte[] bytes) { + // Assume the bytes are an image. So, convert the bytes to a base64 encoded + // following the prefix pattern. + return String.format("data:%s;base64,%s", mimeType.toString(), Base64.getEncoder().encodeToString(bytes)); + } + else if (mediaContentData instanceof String text) { + // Assume the text is a URLs or a base64 encoded image prefixed by the user. + return text; + } + else { + throw new IllegalArgumentException( + "Unsupported media data type: " + mediaContentData.getClass().getSimpleName()); + } + } + private List getFunctionTools(Set functionNames) { return this.resolveFunctionCallbacks(functionNames).stream().map(functionCallback -> { var function = new MistralAiApi.FunctionTool.Function(functionCallback.getDescription(), diff --git a/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/api/MistralAiApi.java b/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/api/MistralAiApi.java index 4743b0e03..433ace72b 100644 --- a/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/api/MistralAiApi.java +++ b/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/api/MistralAiApi.java @@ -740,7 +740,8 @@ public class MistralAiApi { /** * Message comprising the conversation. * - * @param content The contents of the message. + * @param rawContent The contents of the message. Can be either a {@link MediaContent} + * or a {@link String}. The response message content is always a {@link String}. * @param role The role of the messages author. Could be one of the {@link Role} * types. * @param name The name of the author of the message. @@ -752,7 +753,7 @@ public class MistralAiApi { @JsonInclude(Include.NON_NULL) public record ChatCompletionMessage( // @formatter:off - @JsonProperty("content") String content, + @JsonProperty("content") Object rawContent, @JsonProperty("role") Role role, @JsonProperty("name") String name, @JsonProperty("tool_calls") List toolCalls, @@ -767,7 +768,7 @@ public class MistralAiApi { * @param toolCalls The tool calls generated by the model, such as function calls. * Applicable only for {@link Role#ASSISTANT} role and null otherwise. */ - public ChatCompletionMessage(String content, Role role, String name, List toolCalls) { + public ChatCompletionMessage(Object content, Role role, String name, List toolCalls) { this(content, role, name, toolCalls, null); } @@ -777,10 +778,23 @@ public class MistralAiApi { * @param content The contents of the message. * @param role The role of the author of this message. */ - public ChatCompletionMessage(String content, Role role) { + public ChatCompletionMessage(Object content, Role role) { this(content, role, null, null, null); } + /** + * Get message content as String. + */ + public String content() { + if (this.rawContent == null) { + return null; + } + if (this.rawContent instanceof String text) { + return text; + } + throw new IllegalStateException("The content is not a string!"); + } + /** * The role of the author of this message. * @@ -830,6 +844,63 @@ public class MistralAiApi { } + /** + * An array of content parts with a defined type. Each MediaContent can be of + * either "text" or "image_url" type. Only one option allowed. + * + * @param type Content type, each can be of type text or image_url. + * @param text The text content of the message. + * @param imageUrl The image content of the message. + */ + @JsonInclude(Include.NON_NULL) + public record MediaContent( + // @formatter:off + @JsonProperty("type") String type, + @JsonProperty("text") String text, + @JsonProperty("image_url") ImageUrl imageUrl + // @formatter:on + ) { + + /** + * Shortcut constructor for a text content. + * @param text The text content of the message. + */ + public MediaContent(String text) { + this("text", text, null); + } + + /** + * Shortcut constructor for an image content. + * @param imageUrl The image content of the message. + */ + public MediaContent(ImageUrl imageUrl) { + this("image_url", null, imageUrl); + } + + /** + * Shortcut constructor for an image content. + * + * @param url Either a URL of the image or the base64 encoded image data. The + * base64 encoded image data must have a special prefix in the following + * format: "data:{mimetype};base64,{base64-encoded-image-data}". + * @param detail Specifies the detail level of the image. + */ + @JsonInclude(Include.NON_NULL) + public record ImageUrl( + // @formatter:off + @JsonProperty("url") String url, + @JsonProperty("detail") String detail + // @formatter:on + ) { + + public ImageUrl(String url) { + this(url, null); + } + + } + + } + } /** diff --git a/models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatModelIT.java b/models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatModelIT.java index 797dd8b96..45344d43b 100644 --- a/models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatModelIT.java +++ b/models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatModelIT.java @@ -16,18 +16,12 @@ package org.springframework.ai.mistralai; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import reactor.core.publisher.Flux; - import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.UserMessage; @@ -35,6 +29,7 @@ import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.model.StreamingChatModel; +import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.chat.prompt.PromptTemplate; import org.springframework.ai.chat.prompt.SystemPromptTemplate; @@ -42,18 +37,31 @@ import org.springframework.ai.converter.BeanOutputConverter; import org.springframework.ai.converter.ListOutputConverter; import org.springframework.ai.converter.MapOutputConverter; import org.springframework.ai.mistralai.api.MistralAiApi; +import org.springframework.ai.model.Media; import org.springframework.ai.model.function.FunctionCallback; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; +import org.springframework.util.MimeTypeUtils; +import reactor.core.publisher.Flux; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; /** * @author Christian Tzolov * @author Alexandros Pappas + * @author Thomas Vitale * @since 0.8.1 */ @SpringBootTest(classes = MistralAiTestConfiguration.class) @@ -242,9 +250,65 @@ class MistralAiChatModelIT { assertThat(content).containsAnyOf("10.0", "10"); } + @ParameterizedTest(name = "{0} : {displayName} ") + @ValueSource(strings = { "pixtral-large-latest" }) + void multiModalityEmbeddedImage(String modelName) { + var imageData = new ClassPathResource("/test.png"); + + var userMessage = new UserMessage("Explain what do you see on this picture?", + List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData))); + + var response = this.chatModel + .call(new Prompt(List.of(userMessage), ChatOptions.builder().model(modelName).build())); + + logger.info(response.getResult().getOutput().getText()); + assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple"); + assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket", "fruit stand"); + } + + @ParameterizedTest(name = "{0} : {displayName} ") + @ValueSource(strings = { "pixtral-large-latest" }) + void multiModalityImageUrl(String modelName) throws IOException { + var userMessage = new UserMessage("Explain what do you see on this picture?", + List.of(Media.builder() + .mimeType(MimeTypeUtils.IMAGE_PNG) + .data(new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png")) + .build())); + + ChatResponse response = this.chatModel + .call(new Prompt(List.of(userMessage), ChatOptions.builder().model(modelName).build())); + + logger.info(response.getResult().getOutput().getText()); + assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple"); + assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket", "fruit stand"); + } + + @Test + void streamingMultiModalityImageUrl() throws IOException { + var userMessage = new UserMessage("Explain what do you see on this picture?", + List.of(Media.builder() + .mimeType(MimeTypeUtils.IMAGE_PNG) + .data(new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png")) + .build())); + + Flux response = this.streamingChatModel.stream(new Prompt(List.of(userMessage), + ChatOptions.builder().model(MistralAiApi.ChatModel.PIXTRAL_LARGE.getValue()).build())); + + String content = response.collectList() + .block() + .stream() + .map(ChatResponse::getResults) + .flatMap(List::stream) + .map(Generation::getOutput) + .map(AssistantMessage::getText) + .collect(Collectors.joining()); + logger.info("Response: {}", content); + assertThat(content).contains("bananas", "apple"); + assertThat(content).containsAnyOf("bowl", "basket", "fruit stand"); + } + @Test void streamFunctionCallUsageTest() { - UserMessage userMessage = new UserMessage( "What's the weather like in San Francisco, Tokyo, and Paris? Response in Celsius"); diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/mistralai-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/mistralai-chat.adoc index 91a4cfe77..b34eaa628 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/mistralai-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/mistralai-chat.adoc @@ -140,6 +140,62 @@ You can register custom Java functions with the `MistralAiChatModel` and have th This is a powerful technique to connect the LLM capabilities with external tools and APIs. Read more about xref:api/chat/functions/mistralai-chat-functions.adoc[Mistral AI Function Calling]. +== Multimodal + +Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. +Mistral AI supports text and vision modalities. + +=== Vision + +Mistral AI models that offer vision multimodal support include `pixtral-large-latest`. +Refer to the link:https://docs.mistral.ai/capabilities/vision/[Vision] guide for more information. + +The Mistral AI link:https://docs.mistral.ai/api/#tag/chat/operation/chat_completion_v1_chat_completions_post[User Message API] can incorporate a list of base64-encoded images or image urls with the message. +Spring AI’s link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java[Message] interface facilitates multimodal AI models by introducing the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/model/Media.java[Media] type. +This type encompasses data and details regarding media attachments in messages, utilizing Spring’s `org.springframework.util.MimeType` and a `org.springframework.core.io.Resource` for the raw media data. + +Below is a code example excerpted from `MistralAiChatModelIT.java`, illustrating the fusion of user text with an image. + +[source,java] +---- +var imageResource = new ClassPathResource("/multimodal.test.png"); + +var userMessage = new UserMessage("Explain what do you see on this picture?", + new Media(MimeTypeUtils.IMAGE_PNG, this.imageResource)); + +ChatResponse response = chatModel.call(new Prompt(this.userMessage, + ChatOptions.builder().model(MistralAiApi.ChatModel.PIXTRAL_LARGE.getValue()).build())); +---- + +or the image URL equivalent: + +[source,java] +---- +var userMessage = new UserMessage("Explain what do you see on this picture?", + new Media(MimeTypeUtils.IMAGE_PNG, + "https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png")); + +ChatResponse response = chatModel.call(new Prompt(this.userMessage, + ChatOptions.builder().model(MistralAiApi.ChatModel.PIXTRAL_LARGE.getValue()).build())); +---- + +TIP: You can pass multiple images as well. + +The example shows a model taking as an input the `multimodal.test.png` image: + +image::multimodal.test.png[Multimodal Test Image, 200, 200, align="left"] + +along with the text message "Explain what do you see on this picture?", and generating a response like this: + +---- +This is an image of a fruit bowl with a simple design. The bowl is made of metal with curved wire edges that +create an open structure, allowing the fruit to be visible from all angles. Inside the bowl, there are two +yellow bananas resting on top of what appears to be a red apple. The bananas are slightly overripe, as +indicated by the brown spots on their peels. The bowl has a metal ring at the top, likely to serve as a handle +for carrying. The bowl is placed on a flat surface with a neutral-colored background that provides a clear +view of the fruit inside. +---- + == OpenAI API Compatibility Mistral is OpenAI API-compatible and you can use the xref:api/chat/openai-chat.adoc[Spring AI OpenAI] client to talk to Mistrial. diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc index ec2a986a8..1829b8c15 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc @@ -65,9 +65,10 @@ and produce a response like: Spring AI provides multimodal support for the following chat models: -* xref:api/chat/openai-chat.adoc#_multimodal[OpenAI (e.g. GPT-4 and GPT-4o models)] -* xref:api/chat/ollama-chat.adoc#_multimodal[Ollama (e.g. LlaVa, Baklava, Llama3.2 models)] -* xref:api/chat/vertexai-gemini-chat.adoc#_multimodal[Vertex AI Gemini (e.g. gemini-1.5-pro-001, gemini-1.5-flash-001 models)] * xref:api/chat/anthropic-chat.adoc#_multimodal[Anthropic Claude 3] * xref:api/chat/bedrock-converse.adoc#_multimodal[AWS Bedrock Converse] * xref:api/chat/azure-openai-chat.adoc#_multimodal[Azure Open AI (e.g. GPT-4o models)] +* xref:api/chat/mistralai-chat.adoc#_multimodal[Mistral AI (e.g. Mistral Pixtral models)] +* xref:api/chat/ollama-chat.adoc#_multimodal[Ollama (e.g. LlaVa, Baklava, Llama3.2 models)] +* xref:api/chat/openai-chat.adoc#_multimodal[OpenAI (e.g. GPT-4 and GPT-4o models)] +* xref:api/chat/vertexai-gemini-chat.adoc#_multimodal[Vertex AI Gemini (e.g. gemini-1.5-pro-001, gemini-1.5-flash-001 models)]