From c9dd336ea399eae12f28ad93516f78a8f200a4e6 Mon Sep 17 00:00:00 2001 From: Benoit Moussaud Date: Thu, 6 Jun 2024 09:43:30 +0200 Subject: [PATCH] Add multimodality support for Azure OpenAI API - update Azure OpenAI documentation. - add ITs --- .../ai/azure/openai/AzureOpenAiChatModel.java | 43 ++++++++++----- .../azure/openai/AzureOpenAiChatModelIT.java | 53 ++++++++++++++----- .../openai/image/AzureOpenAiImageModelIT.java | 8 ++- .../pages/api/chat/azure-openai-chat.adoc | 38 +++++++++++++ .../modules/ROOT/pages/api/multimodality.adoc | 3 +- 5 files changed, 115 insertions(+), 30 deletions(-) diff --git a/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java index f1d2a06c0..b7e4535aa 100644 --- a/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java +++ b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java @@ -20,9 +20,16 @@ import com.azure.ai.openai.models.ChatChoice; import com.azure.ai.openai.models.ChatCompletions; import com.azure.ai.openai.models.ChatCompletionsFunctionToolCall; import com.azure.ai.openai.models.ChatCompletionsFunctionToolDefinition; +import com.azure.ai.openai.models.ChatCompletionsJsonResponseFormat; import com.azure.ai.openai.models.ChatCompletionsOptions; +import com.azure.ai.openai.models.ChatCompletionsResponseFormat; +import com.azure.ai.openai.models.ChatCompletionsTextResponseFormat; import com.azure.ai.openai.models.ChatCompletionsToolCall; import com.azure.ai.openai.models.ChatCompletionsToolDefinition; +import com.azure.ai.openai.models.ChatMessageContentItem; +import com.azure.ai.openai.models.ChatMessageImageContentItem; +import com.azure.ai.openai.models.ChatMessageImageUrl; +import com.azure.ai.openai.models.ChatMessageTextContentItem; import com.azure.ai.openai.models.ChatRequestAssistantMessage; import com.azure.ai.openai.models.ChatRequestMessage; import com.azure.ai.openai.models.ChatRequestSystemMessage; @@ -32,23 +39,19 @@ import com.azure.ai.openai.models.CompletionsFinishReason; import com.azure.ai.openai.models.ContentFilterResultsForPrompt; import com.azure.ai.openai.models.FunctionCall; import com.azure.ai.openai.models.FunctionDefinition; -import com.azure.ai.openai.models.ChatCompletionsJsonResponseFormat; -import com.azure.ai.openai.models.ChatCompletionsTextResponseFormat; -import com.azure.ai.openai.models.ChatCompletionsResponseFormat; import com.azure.core.util.BinaryData; import com.azure.core.util.IterableStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.springframework.ai.azure.openai.metadata.AzureOpenAiChatResponseMetadata; -import org.springframework.ai.chat.model.ChatModel; -import org.springframework.ai.chat.model.ChatResponse; -import org.springframework.ai.chat.model.Generation; -import org.springframework.ai.chat.model.StreamingChatModel; import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.metadata.ChatGenerationMetadata; import org.springframework.ai.chat.metadata.PromptMetadata; import org.springframework.ai.chat.metadata.PromptMetadata.PromptFilterMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.model.StreamingChatModel; import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.model.ModelOptionsUtils; @@ -58,6 +61,7 @@ import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; import reactor.core.publisher.Flux; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -74,6 +78,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * @author John Blum * @author Christian Tzolov * @author Grogdunn + * @author Benoit Moussaud * @see ChatModel * @see com.azure.ai.openai.OpenAIClient */ @@ -87,16 +92,16 @@ public class AzureOpenAiChatModel private final Logger logger = LoggerFactory.getLogger(getClass()); - /** - * The configuration information for a chat completions request. - */ - private AzureOpenAiChatOptions defaultOptions; - /** * The {@link OpenAIClient} used to interact with the Azure OpenAI service. */ private final OpenAIClient openAIClient; + /** + * The configuration information for a chat completions request. + */ + private AzureOpenAiChatOptions defaultOptions; + public AzureOpenAiChatModel(OpenAIClient microsoftOpenAiClient) { this(microsoftOpenAiClient, AzureOpenAiChatOptions.builder() @@ -277,7 +282,17 @@ public class AzureOpenAiChatModel switch (message.getMessageType()) { case USER: - return new ChatRequestUserMessage(message.getContent()); + // https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/openai/azure-ai-openai/README.md#text-completions-with-images + List items = new ArrayList<>(); + items.add(new ChatMessageTextContentItem(message.getContent())); + if (!CollectionUtils.isEmpty(message.getMedia())) { + items.addAll(message.getMedia() + .stream() + .map(media -> new ChatMessageImageContentItem( + new ChatMessageImageUrl(media.getData().toString()))) + .toList()); + } + return new ChatRequestUserMessage(items); case SYSTEM: return new ChatRequestSystemMessage(message.getContent()); case ASSISTANT: diff --git a/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java b/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java index bd9d3c804..079b840e2 100644 --- a/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java +++ b/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java @@ -15,23 +15,26 @@ */ package org.springframework.ai.azure.openai; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.URL; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import com.azure.ai.openai.OpenAIClient; -import com.azure.ai.openai.OpenAIClientBuilder; -import com.azure.core.credential.AzureKeyCredential; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; - -import org.springframework.ai.chat.model.ChatResponse; -import org.springframework.ai.chat.model.Generation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.chat.prompt.PromptTemplate; import org.springframework.ai.chat.prompt.SystemPromptTemplate; @@ -43,20 +46,22 @@ import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.Bean; import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.util.MimeTypeUtils; -import static org.assertj.core.api.Assertions.assertThat; +import com.azure.ai.openai.OpenAIClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; @SpringBootTest(classes = AzureOpenAiChatModelIT.TestConfiguration.class) @EnabledIfEnvironmentVariable(named = "AZURE_OPENAI_API_KEY", matches = ".+") @EnabledIfEnvironmentVariable(named = "AZURE_OPENAI_ENDPOINT", matches = ".+") class AzureOpenAiChatModelIT { + private static final Logger logger = LoggerFactory.getLogger(AzureOpenAiChatModelIT.class); + @Autowired private AzureOpenAiChatModel chatModel; - record ActorsFilms(String actor, List movies) { - } - @Test void roleTest() { Message systemMessage = new SystemPromptTemplate(""" @@ -130,9 +135,6 @@ class AzureOpenAiChatModelIT { assertThat(actorsFilms.actor()).isNotNull(); } - record ActorsFilmsRecord(String actor, List movies) { - } - @Test void beanOutputConverterRecords() { @@ -183,6 +185,31 @@ class AzureOpenAiChatModelIT { assertThat(actorsFilms.movies()).hasSize(5); } + @Test + void multiModalityImageUrl() throws IOException { + + // TODO: add url method that wrapps the checked exception. + URL url = new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png"); + + // @formatter:off + String response = ChatClient.create(chatModel).prompt() + .options(AzureOpenAiChatOptions.builder().withDeploymentName("gpt-4o").build()) + .user(u -> u.text("Explain what do you see on this picture?").media(MimeTypeUtils.IMAGE_PNG, url)) + .call() + .content(); + // @formatter:on + + logger.info(response); + assertThat(response).contains("bananas", "apple"); + assertThat(response).containsAnyOf("bowl", "basket"); + } + + record ActorsFilms(String actor, List movies) { + } + + record ActorsFilmsRecord(String actor, List movies) { + } + @SpringBootConfiguration public static class TestConfiguration { diff --git a/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/image/AzureOpenAiImageModelIT.java b/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/image/AzureOpenAiImageModelIT.java index 4aff0c673..f57dfb6a7 100644 --- a/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/image/AzureOpenAiImageModelIT.java +++ b/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/image/AzureOpenAiImageModelIT.java @@ -9,12 +9,16 @@ import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.springframework.ai.azure.openai.AzureOpenAiImageModel; import org.springframework.ai.azure.openai.AzureOpenAiImageOptions; import org.springframework.ai.azure.openai.metadata.AzureOpenAiImageGenerationMetadata; -import org.springframework.ai.image.*; +import org.springframework.ai.image.Image; +import org.springframework.ai.image.ImageModel; +import org.springframework.ai.image.ImageOptionsBuilder; +import org.springframework.ai.image.ImagePrompt; +import org.springframework.ai.image.ImageResponse; +import org.springframework.ai.image.ImageResponseMetadata; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.Bean; -import org.springframework.util.StringUtils; import static org.assertj.core.api.Assertions.assertThat; diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc index 0aef13095..d5af1e2c2 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc @@ -142,6 +142,44 @@ You can register custom Java functions with the AzureOpenAiChatModel and have th This is a powerful technique to connect the LLM capabilities with external tools and APIs. Read more about xref:api/chat/functions/azure-open-ai-chat-functions.adoc[Azure OpenAI Function Calling]. +== Multimodal + +Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. +Presently, the Azure OpenAI `gpt-4o` model offers multimodal support. + +The Azure OpenAI can incorporate a list of base64-encoded images or image urls with the message. +Spring AI’s link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java[Message] interface facilitates multimodal AI models by introducing the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java[Media] type. +This type encompasses data and details regarding media attachments in messages, utilizing Spring’s `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data. + +Below is a code example excerpted from link:https://github.com/spring-projects/spring-ai/blob/b3cfa2b900ea785e055e4ff71086eeb52f6578a3/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java[OpenAiChatModelIT.java], illustrating the fusion of user text with an image using the the `GPT_4_VISION_PREVIEW` model. + +[source,java] +---- +URL url = new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png"); +String response = ChatClient.create(chatModel).prompt() + .options(AzureOpenAiChatOptions.builder().withDeploymentName("gpt4o").build()) + .user(u -> u.text("Explain what do you see on this picture?").media(MimeTypeUtils.IMAGE_PNG, url)) + .call() + .content(); +---- + +TIP: you can pass multiple images as well. + +It takes as an input the `multimodal.test.png` image: + +image::multimodal.test.png[Multimodal Test Image, 200, 200, align="left"] + +along with the text message "Explain what do you see on this picture?", and generates a response like this: + +---- +This is an image of a fruit bowl with a simple design. The bowl is made of metal with curved wire edges that +create an open structure, allowing the fruit to be visible from all angles. Inside the bowl, there are two +yellow bananas resting on top of what appears to be a red apple. The bananas are slightly overripe, as +indicated by the brown spots on their peels. The bowl has a metal ring at the top, likely to serve as a handle +for carrying. The bowl is placed on a flat surface with a neutral-colored background that provides a clear +view of the fruit inside. +---- + == Sample Controller https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-azure-openai-spring-boot-starter` to your pom (or gradle) dependencies. diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc index c7ddbc3fd..f2db074d3 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc @@ -73,4 +73,5 @@ Latest version of Spring AI provides multimodal support for the following Chat C * xref:api/chat/ollama-chat.adoc#_multimodal[Ollama - (LlaVa and Baklava models)] * xref:api/chat/vertexai-gemini-chat.adoc#_multimodal[Vertex AI Gemini - (gemini-pro-vision model)] * xref:api/chat/anthropic-chat.adoc#_multimodal[Anthropic Claude 3] -* xref:api/chat/bedrock/bedrock-anthropic3.adoc#_multimodal[AWS Bedrock Anthropic Claude 3] \ No newline at end of file +* xref:api/chat/bedrock/bedrock-anthropic3.adoc#_multimodal[AWS Bedrock Anthropic Claude 3] +* xref:api/chat/azure-openai-chat.adoc#_multimodal[Azure Open AI - (GPT-4o models)] \ No newline at end of file