Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update javadoc for message package and docs for Gemini Multimodal support

Browse Source
This commit is contained in:
Mark Pollack
2024-03-11 19:14:53 -04:00
parent c5d9ae3648
commit 7f1570d722
11 changed files with 73 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClient.java
View File

@@ -333,7 +333,7 @@ public class VertexAiGeminiChatClient
List<Part> parts = new ArrayList<>(List.of(textPart));
List<Part> mediaParts = userMessage.getMediaData()
List<Part> mediaParts = userMessage.getMedia()
.stream()
.map(mediaData -> PartMaker.fromMimeTypeAndData(mediaData.getMimeType().toString(),
mediaData.getData()))

6
models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java
View File

@@ -28,7 +28,7 @@ import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.MediaData;
import org.springframework.ai.chat.messages.Media;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.prompt.Prompt;
@@ -189,7 +189,7 @@ class VertexAiGeminiChatClientIT {
byte[] data = new ClassPathResource("/vertex.test.png").getContentAsByteArray();
var userMessage = new UserMessage("Explain what do you see o this picture?",
List.of(new MediaData(MimeTypeUtils.IMAGE_PNG, data)));
List.of(new Media(MimeTypeUtils.IMAGE_PNG, data)));
ChatResponse response = client.call(new Prompt(List.of(userMessage)));
@@ -208,7 +208,7 @@ class VertexAiGeminiChatClientIT {
// "https://storage.googleapis.com/github-repo/img/gemini/multimodality_usecases_overview/banana-apple.jpg";
// userMessage = new UserMessage("Explain what do you see o this picture?",
// List.of(new MediaData(MimeTypeDetector.getMimeType(imageUrl), imageUrl)));
// List.of(new Media(MimeTypeDetector.getMimeType(imageUrl), imageUrl)));
// response = client.call(new Prompt(List.of(userMessage)));
// assertThat(response.getResult().getOutput().getContent()).contains("bananas",

16
spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AbstractMessage.java
View File

@@ -27,13 +27,20 @@ import org.springframework.core.io.Resource;
import org.springframework.util.Assert;
import org.springframework.util.StreamUtils;
/**
* The AbstractMessage class is an abstract implementation of the Message interface. It
* provides a base implementation for message content, media attachments, properties, and
* message type.
*
* @see Message
*/
public abstract class AbstractMessage implements Message {
protected final MessageType messageType;
protected final String textContent;
protected final List<MediaData> mediaData;
protected final List<Media> mediaData;
/**
* Additional options for the message to influence the response, not a generative map.
@@ -46,18 +53,17 @@ public abstract class AbstractMessage implements Message {
protected AbstractMessage(MessageType messageType, String content, Map<String, Object> messageProperties) {
Assert.notNull(messageType, "Message type must not be null");
// Assert.notNull(content, "Content must not be null");
this.messageType = messageType;
this.textContent = content;
this.mediaData = new ArrayList<>();
this.properties = messageProperties;
}
protected AbstractMessage(MessageType messageType, String textContent, List<MediaData> mediaData) {
protected AbstractMessage(MessageType messageType, String textContent, List<Media> mediaData) {
this(messageType, textContent, mediaData, Map.of());
}
protected AbstractMessage(MessageType messageType, String textContent, List<MediaData> mediaData,
protected AbstractMessage(MessageType messageType, String textContent, List<Media> mediaData,
Map<String, Object> messageProperties) {
Assert.notNull(messageType, "Message type must not be null");
@@ -97,7 +103,7 @@ public abstract class AbstractMessage implements Message {
}
@Override
public List<MediaData> getMediaData() {
public List<Media> getMedia() {
return this.mediaData;
}

4
spring-ai-core/src/main/java/org/springframework/ai/chat/messages/ChatMessage.java
View File

@@ -17,6 +17,10 @@ package org.springframework.ai.chat.messages;
import java.util.Map;
/**
* Represents a chat message in a chat application.
*
*/
public class ChatMessage extends AbstractMessage {
public ChatMessage(String role, String content) {

4
spring-ai-core/src/main/java/org/springframework/ai/chat/messages/FunctionMessage.java
View File

@@ -17,6 +17,10 @@ package org.springframework.ai.chat.messages;
import java.util.Map;
/**
* The FunctionMessage class represents a message with a function content in a chat
* application.
*/
public class FunctionMessage extends AbstractMessage {
public FunctionMessage(String content) {

11
spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MediaData.java → spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java
View File

@@ -19,17 +19,22 @@ import org.springframework.util.Assert;
import org.springframework.util.MimeType;
/**
* The Media class represents the data and metadata of a media attachment in a message. It
* consists of a MIME type and the raw data.
*
* This class is used as a parameter in the constructor of the UserMessage class.
*
* @author Christian Tzolov
* @since 0.8.1
*/
public class MediaData {
public class Media {
private final MimeType mimeType;
private final Object data;
public MediaData(MimeType mimeType, Object data) {
public Media(MimeType mimeType, Object data) {
Assert.notNull(mimeType, "MimeType must not be null");
// Assert.notNull(data, "Data must not be null");
this.mimeType = mimeType;
this.data = data;
}

10
spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java
View File

@@ -18,11 +18,19 @@ package org.springframework.ai.chat.messages;
import java.util.List;
import java.util.Map;
/**
* The Message interface represents a message that can be sent or received in a chat
* application. Messages can have content, media attachments, properties, and message
* types.
*
* @see Media
* @see MessageType
*/
public interface Message {
String getContent();
List<MediaData> getMediaData();
List<Media> getMedia();
Map<String, Object> getProperties();

4
spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MessageType.java
View File

@@ -15,6 +15,10 @@
*/
package org.springframework.ai.chat.messages;
/**
* The MessageType enum represents the type of a message in a chat application. It can be
* one of the following: USER, ASSISTANT, SYSTEM, FUNCTION.
*/
public enum MessageType {
USER("user"),

4
spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java
View File

@@ -34,8 +34,8 @@ public class UserMessage extends AbstractMessage {
super(MessageType.USER, resource);
}
public UserMessage(String textContent, List<MediaData> mediaDataList) {
super(MessageType.USER, textContent, mediaDataList);
public UserMessage(String textContent, List<Media> mediaList) {
super(MessageType.USER, textContent, mediaList);
}
@Override

23
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/vertexai-gemini-chat.adoc
View File

@@ -107,6 +107,26 @@ You can register custom Java functions with the VertexAiGeminiChatClient and hav
This is a powerful technique to connect the LLM capabilities with external tools and APIs.
Read more about xref:api/clients/functions/vertexai-gemini-chat-functions.adoc[Vertex AI Gemini Function Calling].
=== Multimodal Example
Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. This paradigm represents a significant advancement in AI models.
Google's Gemini AI models support this capability by comprehending and integrating text, code, audio, images, and video. For more details, refer to the blog post [Introducing Gemini](https://blog.google/technology/ai/google-gemini-ai/#introducing-gemini).
Spring AI's `Message` interface supports multimodal AI models by introducing the Media type.
This type contains data and information about media attachments in messages, using Spring's `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data.
Below is a simple code example extracted from [VertexAiGeminiChatClientIT.java](https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java), demonstrating the combination of user text with an image.
[source,java]
----
byte[] data = new ClassPathResource("/vertex-test.png").getContentAsByteArray();
var userMessage = new UserMessage("Explain what do you see o this picture?",
List.of(new Media(MimeTypeUtils.IMAGE_PNG, data)));
ChatResponse response = chatClient.call(new Prompt(List.of(userMessage)));
----
=== Sample Controller (Auto-configuration)
@@ -195,6 +215,3 @@ ChatResponse response = chatClient.call(
The `VertexAiGeminiChatOptions` provides the configuration information for the chat requests.
The `VertexAiGeminiChatOptions.Builder` is fluent options builder.
=== Multi-modal
https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java[VertexAiGeminiChatClientIT.java]

11
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/prompt.adoc
View File

@@ -45,16 +45,19 @@ public class Prompt {
=== Message
The `Message` interface encapsulates a textual message, a collection of attributes as a `Map`, and a categorization known as `MessageType`. The interface is defined as follows:
The `Message` interface encapsulates a textual message, a collection of attributes as a `Map`, a categorization known as `MessageType`, and a list of media objects for those models that are multimodal.
The interface is defined as follows:
```java
public interface Message {
String getContent();
String getContent();
Map<String, Object> getProperties();
List<Media> getMedia();
MessageType getMessageType();
Map<String, Object> getProperties();
MessageType getMessageType();
}
```