diff --git a/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClient.java b/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClient.java index 2a742d3cd..1641bc9ac 100644 --- a/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClient.java +++ b/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClient.java @@ -333,7 +333,7 @@ public class VertexAiGeminiChatClient List parts = new ArrayList<>(List.of(textPart)); - List mediaParts = userMessage.getMediaData() + List mediaParts = userMessage.getMedia() .stream() .map(mediaData -> PartMaker.fromMimeTypeAndData(mediaData.getMimeType().toString(), mediaData.getData())) diff --git a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java index 59fe0e1ee..8860dc9f9 100644 --- a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java +++ b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java @@ -28,7 +28,7 @@ import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.springframework.ai.chat.ChatResponse; import org.springframework.ai.chat.Generation; import org.springframework.ai.chat.messages.AssistantMessage; -import org.springframework.ai.chat.messages.MediaData; +import org.springframework.ai.chat.messages.Media; import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.prompt.Prompt; @@ -189,7 +189,7 @@ class VertexAiGeminiChatClientIT { byte[] data = new ClassPathResource("/vertex.test.png").getContentAsByteArray(); var userMessage = new UserMessage("Explain what do you see o this picture?", - List.of(new MediaData(MimeTypeUtils.IMAGE_PNG, data))); + List.of(new Media(MimeTypeUtils.IMAGE_PNG, data))); ChatResponse response = client.call(new Prompt(List.of(userMessage))); @@ -208,7 +208,7 @@ class VertexAiGeminiChatClientIT { // "https://storage.googleapis.com/github-repo/img/gemini/multimodality_usecases_overview/banana-apple.jpg"; // userMessage = new UserMessage("Explain what do you see o this picture?", - // List.of(new MediaData(MimeTypeDetector.getMimeType(imageUrl), imageUrl))); + // List.of(new Media(MimeTypeDetector.getMimeType(imageUrl), imageUrl))); // response = client.call(new Prompt(List.of(userMessage))); // assertThat(response.getResult().getOutput().getContent()).contains("bananas", diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AbstractMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AbstractMessage.java index df14f75ef..77b54afea 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AbstractMessage.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AbstractMessage.java @@ -27,13 +27,20 @@ import org.springframework.core.io.Resource; import org.springframework.util.Assert; import org.springframework.util.StreamUtils; +/** + * The AbstractMessage class is an abstract implementation of the Message interface. It + * provides a base implementation for message content, media attachments, properties, and + * message type. + * + * @see Message + */ public abstract class AbstractMessage implements Message { protected final MessageType messageType; protected final String textContent; - protected final List mediaData; + protected final List mediaData; /** * Additional options for the message to influence the response, not a generative map. @@ -46,18 +53,17 @@ public abstract class AbstractMessage implements Message { protected AbstractMessage(MessageType messageType, String content, Map messageProperties) { Assert.notNull(messageType, "Message type must not be null"); - // Assert.notNull(content, "Content must not be null"); this.messageType = messageType; this.textContent = content; this.mediaData = new ArrayList<>(); this.properties = messageProperties; } - protected AbstractMessage(MessageType messageType, String textContent, List mediaData) { + protected AbstractMessage(MessageType messageType, String textContent, List mediaData) { this(messageType, textContent, mediaData, Map.of()); } - protected AbstractMessage(MessageType messageType, String textContent, List mediaData, + protected AbstractMessage(MessageType messageType, String textContent, List mediaData, Map messageProperties) { Assert.notNull(messageType, "Message type must not be null"); @@ -97,7 +103,7 @@ public abstract class AbstractMessage implements Message { } @Override - public List getMediaData() { + public List getMedia() { return this.mediaData; } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/ChatMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/ChatMessage.java index 2186446ed..ea4803a59 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/ChatMessage.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/ChatMessage.java @@ -17,6 +17,10 @@ package org.springframework.ai.chat.messages; import java.util.Map; +/** + * Represents a chat message in a chat application. + * + */ public class ChatMessage extends AbstractMessage { public ChatMessage(String role, String content) { diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/FunctionMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/FunctionMessage.java index 270958dea..06485ac57 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/FunctionMessage.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/FunctionMessage.java @@ -17,6 +17,10 @@ package org.springframework.ai.chat.messages; import java.util.Map; +/** + * The FunctionMessage class represents a message with a function content in a chat + * application. + */ public class FunctionMessage extends AbstractMessage { public FunctionMessage(String content) { diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MediaData.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java similarity index 77% rename from spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MediaData.java rename to spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java index 5039019c7..ffe377938 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MediaData.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java @@ -19,17 +19,22 @@ import org.springframework.util.Assert; import org.springframework.util.MimeType; /** + * The Media class represents the data and metadata of a media attachment in a message. It + * consists of a MIME type and the raw data. + * + * This class is used as a parameter in the constructor of the UserMessage class. + * * @author Christian Tzolov + * @since 0.8.1 */ -public class MediaData { +public class Media { private final MimeType mimeType; private final Object data; - public MediaData(MimeType mimeType, Object data) { + public Media(MimeType mimeType, Object data) { Assert.notNull(mimeType, "MimeType must not be null"); - // Assert.notNull(data, "Data must not be null"); this.mimeType = mimeType; this.data = data; } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java index 38dfba48b..77e3b5aba 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java @@ -18,11 +18,19 @@ package org.springframework.ai.chat.messages; import java.util.List; import java.util.Map; +/** + * The Message interface represents a message that can be sent or received in a chat + * application. Messages can have content, media attachments, properties, and message + * types. + * + * @see Media + * @see MessageType + */ public interface Message { String getContent(); - List getMediaData(); + List getMedia(); Map getProperties(); diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MessageType.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MessageType.java index add5b4eae..632b66dc8 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MessageType.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MessageType.java @@ -15,6 +15,10 @@ */ package org.springframework.ai.chat.messages; +/** + * The MessageType enum represents the type of a message in a chat application. It can be + * one of the following: USER, ASSISTANT, SYSTEM, FUNCTION. + */ public enum MessageType { USER("user"), diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java index 5a6a596dd..d981a67ed 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java @@ -34,8 +34,8 @@ public class UserMessage extends AbstractMessage { super(MessageType.USER, resource); } - public UserMessage(String textContent, List mediaDataList) { - super(MessageType.USER, textContent, mediaDataList); + public UserMessage(String textContent, List mediaList) { + super(MessageType.USER, textContent, mediaList); } @Override diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/vertexai-gemini-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/vertexai-gemini-chat.adoc index 3afc7975e..b519d6c17 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/vertexai-gemini-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/vertexai-gemini-chat.adoc @@ -107,6 +107,26 @@ You can register custom Java functions with the VertexAiGeminiChatClient and hav This is a powerful technique to connect the LLM capabilities with external tools and APIs. Read more about xref:api/clients/functions/vertexai-gemini-chat-functions.adoc[Vertex AI Gemini Function Calling]. +=== Multimodal Example +Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. This paradigm represents a significant advancement in AI models. + +Google's Gemini AI models support this capability by comprehending and integrating text, code, audio, images, and video. For more details, refer to the blog post [Introducing Gemini](https://blog.google/technology/ai/google-gemini-ai/#introducing-gemini). + +Spring AI's `Message` interface supports multimodal AI models by introducing the Media type. +This type contains data and information about media attachments in messages, using Spring's `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data. + +Below is a simple code example extracted from [VertexAiGeminiChatClientIT.java](https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java), demonstrating the combination of user text with an image. + + +[source,java] +---- +byte[] data = new ClassPathResource("/vertex-test.png").getContentAsByteArray(); + +var userMessage = new UserMessage("Explain what do you see o this picture?", + List.of(new Media(MimeTypeUtils.IMAGE_PNG, data))); + +ChatResponse response = chatClient.call(new Prompt(List.of(userMessage))); +---- === Sample Controller (Auto-configuration) @@ -195,6 +215,3 @@ ChatResponse response = chatClient.call( The `VertexAiGeminiChatOptions` provides the configuration information for the chat requests. The `VertexAiGeminiChatOptions.Builder` is fluent options builder. -=== Multi-modal - -https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java[VertexAiGeminiChatClientIT.java] \ No newline at end of file diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/prompt.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/prompt.adoc index 39290a106..054d829bd 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/prompt.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/prompt.adoc @@ -45,16 +45,19 @@ public class Prompt { === Message -The `Message` interface encapsulates a textual message, a collection of attributes as a `Map`, and a categorization known as `MessageType`. The interface is defined as follows: +The `Message` interface encapsulates a textual message, a collection of attributes as a `Map`, a categorization known as `MessageType`, and a list of media objects for those models that are multimodal. +The interface is defined as follows: ```java public interface Message { - String getContent(); + String getContent(); - Map getProperties(); + List getMedia(); - MessageType getMessageType(); + Map getProperties(); + + MessageType getMessageType(); } ```