Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Mistral: Support Vision Modality

Browse Source 
Introduce multimodality support for Mistral AI, which currently supports text and vision modalities.
Added integration tests and documentation for the new capability.

Signed-off-by: Thomas Vitale <ThomasVitale@users.noreply.github.com>
This commit is contained in:
Thomas Vitale
2024-12-18 12:32:07 +01:00
committed by Christian Tzolov
parent 2e5ee4316f
commit f2820cd8af
5 changed files with 246 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java
View File

@@ -16,6 +16,8 @@
package org.springframework.ai.mistralai;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -27,6 +29,8 @@ import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.contextpropagation.ObservationThreadLocalAccessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.model.Media;
import org.springframework.util.MimeType;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@@ -353,8 +357,19 @@ public class MistralAiChatModel extends AbstractToolCallSupport implements ChatM
List<ChatCompletionMessage> chatCompletionMessages = prompt.getInstructions().stream().map(message -> {
if (message instanceof UserMessage userMessage) {
return List.of(new MistralAiApi.ChatCompletionMessage(userMessage.getText(),
MistralAiApi.ChatCompletionMessage.Role.USER));
Object content = message.getText();
if (!CollectionUtils.isEmpty(userMessage.getMedia())) {
List<ChatCompletionMessage.MediaContent> contentList = new ArrayList<>(
List.of(new ChatCompletionMessage.MediaContent(message.getText())));
contentList.addAll(userMessage.getMedia().stream().map(this::mapToMediaContent).toList());
content = contentList;
}
return List
.of(new MistralAiApi.ChatCompletionMessage(content, MistralAiApi.ChatCompletionMessage.Role.USER));
}
else if (message instanceof SystemMessage systemMessage) {
return List.of(new MistralAiApi.ChatCompletionMessage(systemMessage.getText(),
@@ -424,6 +439,27 @@ public class MistralAiChatModel extends AbstractToolCallSupport implements ChatM
return request;
}
private ChatCompletionMessage.MediaContent mapToMediaContent(Media media) {
return new ChatCompletionMessage.MediaContent(new ChatCompletionMessage.MediaContent.ImageUrl(
this.fromMediaData(media.getMimeType(), media.getData())));
}
private String fromMediaData(MimeType mimeType, Object mediaContentData) {
if (mediaContentData instanceof byte[] bytes) {
// Assume the bytes are an image. So, convert the bytes to a base64 encoded
// following the prefix pattern.
return String.format("data:%s;base64,%s", mimeType.toString(), Base64.getEncoder().encodeToString(bytes));
}
else if (mediaContentData instanceof String text) {
// Assume the text is a URLs or a base64 encoded image prefixed by the user.
return text;
}
else {
throw new IllegalArgumentException(
"Unsupported media data type: " + mediaContentData.getClass().getSimpleName());
}
}
private List<MistralAiApi.FunctionTool> getFunctionTools(Set<String> functionNames) {
return this.resolveFunctionCallbacks(functionNames).stream().map(functionCallback -> {
var function = new MistralAiApi.FunctionTool.Function(functionCallback.getDescription(),

79
models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/api/MistralAiApi.java
View File

@@ -740,7 +740,8 @@ public class MistralAiApi {
/**
* Message comprising the conversation.
*
* @param content The contents of the message.
* @param rawContent The contents of the message. Can be either a {@link MediaContent}
* or a {@link String}. The response message content is always a {@link String}.
* @param role The role of the messages author. Could be one of the {@link Role}
* types.
* @param name The name of the author of the message.
@@ -752,7 +753,7 @@ public class MistralAiApi {
@JsonInclude(Include.NON_NULL)
public record ChatCompletionMessage(
// @formatter:off
@JsonProperty("content") String content,
@JsonProperty("content") Object rawContent,
@JsonProperty("role") Role role,
@JsonProperty("name") String name,
@JsonProperty("tool_calls") List<ToolCall> toolCalls,
@@ -767,7 +768,7 @@ public class MistralAiApi {
* @param toolCalls The tool calls generated by the model, such as function calls.
* Applicable only for {@link Role#ASSISTANT} role and null otherwise.
*/
public ChatCompletionMessage(String content, Role role, String name, List<ToolCall> toolCalls) {
public ChatCompletionMessage(Object content, Role role, String name, List<ToolCall> toolCalls) {
this(content, role, name, toolCalls, null);
}
@@ -777,10 +778,23 @@ public class MistralAiApi {
* @param content The contents of the message.
* @param role The role of the author of this message.
*/
public ChatCompletionMessage(String content, Role role) {
public ChatCompletionMessage(Object content, Role role) {
this(content, role, null, null, null);
}
/**
* Get message content as String.
*/
public String content() {
if (this.rawContent == null) {
return null;
}
if (this.rawContent instanceof String text) {
return text;
}
throw new IllegalStateException("The content is not a string!");
}
/**
* The role of the author of this message.
*
@@ -830,6 +844,63 @@ public class MistralAiApi {
}
/**
* An array of content parts with a defined type. Each MediaContent can be of
* either "text" or "image_url" type. Only one option allowed.
*
* @param type Content type, each can be of type text or image_url.
* @param text The text content of the message.
* @param imageUrl The image content of the message.
*/
@JsonInclude(Include.NON_NULL)
public record MediaContent(
// @formatter:off
@JsonProperty("type") String type,
@JsonProperty("text") String text,
@JsonProperty("image_url") ImageUrl imageUrl
// @formatter:on
) {
/**
* Shortcut constructor for a text content.
* @param text The text content of the message.
*/
public MediaContent(String text) {
this("text", text, null);
}
/**
* Shortcut constructor for an image content.
* @param imageUrl The image content of the message.
*/
public MediaContent(ImageUrl imageUrl) {
this("image_url", null, imageUrl);
}
/**
* Shortcut constructor for an image content.
*
* @param url Either a URL of the image or the base64 encoded image data. The
* base64 encoded image data must have a special prefix in the following
* format: "data:{mimetype};base64,{base64-encoded-image-data}".
* @param detail Specifies the detail level of the image.
*/
@JsonInclude(Include.NON_NULL)
public record ImageUrl(
// @formatter:off
@JsonProperty("url") String url,
@JsonProperty("detail") String detail
// @formatter:on
) {
public ImageUrl(String url) {
this(url, null);
}
}
}
}
/**

82
models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatModelIT.java
View File

@@ -16,18 +16,12 @@
package org.springframework.ai.mistralai;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.publisher.Flux;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
@@ -35,6 +29,7 @@ import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.model.StreamingChatModel;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.chat.prompt.PromptTemplate;
import org.springframework.ai.chat.prompt.SystemPromptTemplate;
@@ -42,18 +37,31 @@ import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.ai.converter.ListOutputConverter;
import org.springframework.ai.converter.MapOutputConverter;
import org.springframework.ai.mistralai.api.MistralAiApi;
import org.springframework.ai.model.Media;
import org.springframework.ai.model.function.FunctionCallback;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.convert.support.DefaultConversionService;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.util.MimeTypeUtils;
import reactor.core.publisher.Flux;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
* @author Alexandros Pappas
* @author Thomas Vitale
* @since 0.8.1
*/
@SpringBootTest(classes = MistralAiTestConfiguration.class)
@@ -242,9 +250,65 @@ class MistralAiChatModelIT {
assertThat(content).containsAnyOf("10.0", "10");
}
@ParameterizedTest(name = "{0} : {displayName} ")
@ValueSource(strings = { "pixtral-large-latest" })
void multiModalityEmbeddedImage(String modelName) {
var imageData = new ClassPathResource("/test.png");
var userMessage = new UserMessage("Explain what do you see on this picture?",
List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData)));
var response = this.chatModel
.call(new Prompt(List.of(userMessage), ChatOptions.builder().model(modelName).build()));
logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket", "fruit stand");
}
@ParameterizedTest(name = "{0} : {displayName} ")
@ValueSource(strings = { "pixtral-large-latest" })
void multiModalityImageUrl(String modelName) throws IOException {
var userMessage = new UserMessage("Explain what do you see on this picture?",
List.of(Media.builder()
.mimeType(MimeTypeUtils.IMAGE_PNG)
.data(new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png"))
.build()));
ChatResponse response = this.chatModel
.call(new Prompt(List.of(userMessage), ChatOptions.builder().model(modelName).build()));
logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket", "fruit stand");
}
@Test
void streamingMultiModalityImageUrl() throws IOException {
var userMessage = new UserMessage("Explain what do you see on this picture?",
List.of(Media.builder()
.mimeType(MimeTypeUtils.IMAGE_PNG)
.data(new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png"))
.build()));
Flux<ChatResponse> response = this.streamingChatModel.stream(new Prompt(List.of(userMessage),
ChatOptions.builder().model(MistralAiApi.ChatModel.PIXTRAL_LARGE.getValue()).build()));
String content = response.collectList()
.block()
.stream()
.map(ChatResponse::getResults)
.flatMap(List::stream)
.map(Generation::getOutput)
.map(AssistantMessage::getText)
.collect(Collectors.joining());
logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket", "fruit stand");
}
@Test
void streamFunctionCallUsageTest() {
UserMessage userMessage = new UserMessage(
"What's the weather like in San Francisco, Tokyo, and Paris? Response in Celsius");

56
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/mistralai-chat.adoc
View File

@@ -140,6 +140,62 @@ You can register custom Java functions with the `MistralAiChatModel` and have th
This is a powerful technique to connect the LLM capabilities with external tools and APIs.
Read more about xref:api/chat/functions/mistralai-chat-functions.adoc[Mistral AI Function Calling].
== Multimodal
Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats.
Mistral AI supports text and vision modalities.
=== Vision
Mistral AI models that offer vision multimodal support include `pixtral-large-latest`.
Refer to the link:https://docs.mistral.ai/capabilities/vision/[Vision] guide for more information.
The Mistral AI link:https://docs.mistral.ai/api/#tag/chat/operation/chat_completion_v1_chat_completions_post[User Message API] can incorporate a list of base64-encoded images or image urls with the message.
Spring AI’s link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java[Message] interface facilitates multimodal AI models by introducing the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/model/Media.java[Media] type.
This type encompasses data and details regarding media attachments in messages, utilizing Spring’s `org.springframework.util.MimeType` and a `org.springframework.core.io.Resource` for the raw media data.
Below is a code example excerpted from `MistralAiChatModelIT.java`, illustrating the fusion of user text with an image.
[source,java]
----
var imageResource = new ClassPathResource("/multimodal.test.png");
var userMessage = new UserMessage("Explain what do you see on this picture?",
new Media(MimeTypeUtils.IMAGE_PNG, this.imageResource));
ChatResponse response = chatModel.call(new Prompt(this.userMessage,
ChatOptions.builder().model(MistralAiApi.ChatModel.PIXTRAL_LARGE.getValue()).build()));
----
or the image URL equivalent:
[source,java]
----
var userMessage = new UserMessage("Explain what do you see on this picture?",
new Media(MimeTypeUtils.IMAGE_PNG,
"https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png"));
ChatResponse response = chatModel.call(new Prompt(this.userMessage,
ChatOptions.builder().model(MistralAiApi.ChatModel.PIXTRAL_LARGE.getValue()).build()));
----
TIP: You can pass multiple images as well.
The example shows a model taking as an input the `multimodal.test.png` image:
image::multimodal.test.png[Multimodal Test Image, 200, 200, align="left"]
along with the text message "Explain what do you see on this picture?", and generating a response like this:
----
This is an image of a fruit bowl with a simple design. The bowl is made of metal with curved wire edges that
create an open structure, allowing the fruit to be visible from all angles. Inside the bowl, there are two
yellow bananas resting on top of what appears to be a red apple. The bananas are slightly overripe, as
indicated by the brown spots on their peels. The bowl has a metal ring at the top, likely to serve as a handle
for carrying. The bowl is placed on a flat surface with a neutral-colored background that provides a clear
view of the fruit inside.
----
== OpenAI API Compatibility
Mistral is OpenAI API-compatible and you can use the xref:api/chat/openai-chat.adoc[Spring AI OpenAI] client to talk to Mistrial.

7
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc
View File

@@ -65,9 +65,10 @@ and produce a response like:
Spring AI provides multimodal support for the following chat models:
* xref:api/chat/openai-chat.adoc#_multimodal[OpenAI (e.g. GPT-4 and GPT-4o models)]
* xref:api/chat/ollama-chat.adoc#_multimodal[Ollama (e.g. LlaVa, Baklava, Llama3.2 models)]
* xref:api/chat/vertexai-gemini-chat.adoc#_multimodal[Vertex AI Gemini (e.g. gemini-1.5-pro-001, gemini-1.5-flash-001 models)]
* xref:api/chat/anthropic-chat.adoc#_multimodal[Anthropic Claude 3]
* xref:api/chat/bedrock-converse.adoc#_multimodal[AWS Bedrock Converse]
* xref:api/chat/azure-openai-chat.adoc#_multimodal[Azure Open AI (e.g. GPT-4o models)]
* xref:api/chat/mistralai-chat.adoc#_multimodal[Mistral AI (e.g. Mistral Pixtral models)]
* xref:api/chat/ollama-chat.adoc#_multimodal[Ollama (e.g. LlaVa, Baklava, Llama3.2 models)]
* xref:api/chat/openai-chat.adoc#_multimodal[OpenAI (e.g. GPT-4 and GPT-4o models)]
* xref:api/chat/vertexai-gemini-chat.adoc#_multimodal[Vertex AI Gemini (e.g. gemini-1.5-pro-001, gemini-1.5-flash-001 models)]