diff --git a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModelIT.java b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModelIT.java index 4d6a06444..186bb02f3 100644 --- a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModelIT.java +++ b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModelIT.java @@ -49,6 +49,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.core.convert.support.DefaultConversionService; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; +import org.springframework.util.MimeType; import org.springframework.util.MimeTypeUtils; import static org.assertj.core.api.Assertions.assertThat; @@ -246,6 +247,22 @@ class VertexAiGeminiChatModelIT { // https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/intro_multimodal_use_cases.ipynb } + @Test + void multiModalityPdfTest() throws IOException { + + var pdfData = new ClassPathResource("/spring-ai-reference-overview.pdf"); + + var userMessage = new UserMessage( + "You are a very professional document summarization specialist. Please summarize the given document.", + List.of(new Media(new MimeType("application", "pdf"), pdfData))); + + var response = this.chatModel.call(new Prompt(List.of(userMessage))); + + System.out.println(response.getResult().getOutput().getContent()); + + assertThat(response.getResult().getOutput().getContent()).containsAnyOf("Spring AI", "portable API"); + } + record ActorsFilmsRecord(String actor, List movies) { } diff --git a/models/spring-ai-vertex-ai-gemini/src/test/resources/spring-ai-reference-overview.pdf b/models/spring-ai-vertex-ai-gemini/src/test/resources/spring-ai-reference-overview.pdf new file mode 100644 index 000000000..7ff8c7e04 Binary files /dev/null and b/models/spring-ai-vertex-ai-gemini/src/test/resources/spring-ai-reference-overview.pdf differ diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc index 443d53eb4..098a764f9 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc @@ -21,7 +21,7 @@ This table compares various Chat Models supported by Spring AI, detailing their | xref::api/chat/anthropic-chat.adoc[Anthropic Claude] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] | xref::api/chat/azure-openai-chat.adoc[Azure OpenAI] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] -| xref::api/chat/vertexai-gemini-chat.adoc[Google VertexAI Gemini] | text, image, audio, video ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] +| xref::api/chat/vertexai-gemini-chat.adoc[Google VertexAI Gemini] | text, pdf, image, audio, video ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] | xref::api/chat/groq-chat.adoc[Groq (OpenAI-proxy)] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] | xref::api/chat/huggingface.adoc[HuggingFace] | text ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] | xref::api/chat/mistralai-chat.adoc[Mistral AI] | text ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/vertexai-gemini-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/vertexai-gemini-chat.adoc index 58ac42801..80da557ee 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/vertexai-gemini-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/vertexai-gemini-chat.adoc @@ -124,9 +124,11 @@ Read more about xref:api/chat/functions/vertexai-gemini-chat-functions.adoc[Vert == Multimodal -Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. This paradigm represents a significant advancement in AI models. +Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including `text`, `pdf`, `images`, `audio`, and other data formats. +This paradigm represents a significant advancement in AI models. -Google's Gemini AI models support this capability by comprehending and integrating text, code, audio, images, and video. For more details, refer to the blog post https://blog.google/technology/ai/google-gemini-ai/#introducing-gemini[Introducing Gemini]. +Google's Gemini AI models support this capability by comprehending and integrating text, code, audio, images, and video. +For more details, refer to the blog post https://blog.google/technology/ai/google-gemini-ai/#introducing-gemini[Introducing Gemini]. Spring AI's `Message` interface supports multimodal AI models by introducing the Media type. This type contains data and information about media attachments in messages, using Spring's `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data.