docs(vertex-ai-gemini): Add Gemini PDF support docs and add IT

- Add test case for PDF document summarization using Gemini multimodal capabilities
- Update documentation to reflect PDF support in model comparison table
- Add PDF format to multimodal capabilities documentation
This commit is contained in:
Christian Tzolov
2024-11-25 14:09:33 +01:00
parent 87ff101946
commit 979de19162
4 changed files with 22 additions and 3 deletions

View File

@@ -49,6 +49,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.core.convert.support.DefaultConversionService;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.util.MimeType;
import org.springframework.util.MimeTypeUtils;
import static org.assertj.core.api.Assertions.assertThat;
@@ -246,6 +247,22 @@ class VertexAiGeminiChatModelIT {
// https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/intro_multimodal_use_cases.ipynb
}
@Test
void multiModalityPdfTest() throws IOException {
var pdfData = new ClassPathResource("/spring-ai-reference-overview.pdf");
var userMessage = new UserMessage(
"You are a very professional document summarization specialist. Please summarize the given document.",
List.of(new Media(new MimeType("application", "pdf"), pdfData)));
var response = this.chatModel.call(new Prompt(List.of(userMessage)));
System.out.println(response.getResult().getOutput().getContent());
assertThat(response.getResult().getOutput().getContent()).containsAnyOf("Spring AI", "portable API");
}
record ActorsFilmsRecord(String actor, List<String> movies) {
}

View File

@@ -21,7 +21,7 @@ This table compares various Chat Models supported by Spring AI, detailing their
| xref::api/chat/anthropic-chat.adoc[Anthropic Claude] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12]
| xref::api/chat/azure-openai-chat.adoc[Azure OpenAI] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
| xref::api/chat/vertexai-gemini-chat.adoc[Google VertexAI Gemini] | text, image, audio, video ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
| xref::api/chat/vertexai-gemini-chat.adoc[Google VertexAI Gemini] | text, pdf, image, audio, video ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
| xref::api/chat/groq-chat.adoc[Groq (OpenAI-proxy)] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
| xref::api/chat/huggingface.adoc[HuggingFace] | text ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12]
| xref::api/chat/mistralai-chat.adoc[Mistral AI] | text ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]

View File

@@ -124,9 +124,11 @@ Read more about xref:api/chat/functions/vertexai-gemini-chat-functions.adoc[Vert
== Multimodal
Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. This paradigm represents a significant advancement in AI models.
Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including `text`, `pdf`, `images`, `audio`, and other data formats.
This paradigm represents a significant advancement in AI models.
Google's Gemini AI models support this capability by comprehending and integrating text, code, audio, images, and video. For more details, refer to the blog post https://blog.google/technology/ai/google-gemini-ai/#introducing-gemini[Introducing Gemini].
Google's Gemini AI models support this capability by comprehending and integrating text, code, audio, images, and video.
For more details, refer to the blog post https://blog.google/technology/ai/google-gemini-ai/#introducing-gemini[Introducing Gemini].
Spring AI's `Message` interface supports multimodal AI models by introducing the Media type.
This type contains data and information about media attachments in messages, using Spring's `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data.