Add multimodality support for Azure OpenAI API
- update Azure OpenAI documentation. - add ITs
This commit is contained in:
Benoit Moussaud
committed by
Christian Tzolov
Christian Tzolov
parent
958549ecce
commit
c9dd336ea3
@@ -20,9 +20,16 @@ import com.azure.ai.openai.models.ChatChoice;
|
||||
import com.azure.ai.openai.models.ChatCompletions;
|
||||
import com.azure.ai.openai.models.ChatCompletionsFunctionToolCall;
|
||||
import com.azure.ai.openai.models.ChatCompletionsFunctionToolDefinition;
|
||||
import com.azure.ai.openai.models.ChatCompletionsJsonResponseFormat;
|
||||
import com.azure.ai.openai.models.ChatCompletionsOptions;
|
||||
import com.azure.ai.openai.models.ChatCompletionsResponseFormat;
|
||||
import com.azure.ai.openai.models.ChatCompletionsTextResponseFormat;
|
||||
import com.azure.ai.openai.models.ChatCompletionsToolCall;
|
||||
import com.azure.ai.openai.models.ChatCompletionsToolDefinition;
|
||||
import com.azure.ai.openai.models.ChatMessageContentItem;
|
||||
import com.azure.ai.openai.models.ChatMessageImageContentItem;
|
||||
import com.azure.ai.openai.models.ChatMessageImageUrl;
|
||||
import com.azure.ai.openai.models.ChatMessageTextContentItem;
|
||||
import com.azure.ai.openai.models.ChatRequestAssistantMessage;
|
||||
import com.azure.ai.openai.models.ChatRequestMessage;
|
||||
import com.azure.ai.openai.models.ChatRequestSystemMessage;
|
||||
@@ -32,23 +39,19 @@ import com.azure.ai.openai.models.CompletionsFinishReason;
|
||||
import com.azure.ai.openai.models.ContentFilterResultsForPrompt;
|
||||
import com.azure.ai.openai.models.FunctionCall;
|
||||
import com.azure.ai.openai.models.FunctionDefinition;
|
||||
import com.azure.ai.openai.models.ChatCompletionsJsonResponseFormat;
|
||||
import com.azure.ai.openai.models.ChatCompletionsTextResponseFormat;
|
||||
import com.azure.ai.openai.models.ChatCompletionsResponseFormat;
|
||||
import com.azure.core.util.BinaryData;
|
||||
import com.azure.core.util.IterableStream;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.springframework.ai.azure.openai.metadata.AzureOpenAiChatResponseMetadata;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.model.StreamingChatModel;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
|
||||
import org.springframework.ai.chat.metadata.PromptMetadata;
|
||||
import org.springframework.ai.chat.metadata.PromptMetadata.PromptFilterMetadata;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.model.StreamingChatModel;
|
||||
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.model.ModelOptionsUtils;
|
||||
@@ -58,6 +61,7 @@ import org.springframework.util.Assert;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
@@ -74,6 +78,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||
* @author John Blum
|
||||
* @author Christian Tzolov
|
||||
* @author Grogdunn
|
||||
* @author Benoit Moussaud
|
||||
* @see ChatModel
|
||||
* @see com.azure.ai.openai.OpenAIClient
|
||||
*/
|
||||
@@ -87,16 +92,16 @@ public class AzureOpenAiChatModel
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(getClass());
|
||||
|
||||
/**
|
||||
* The configuration information for a chat completions request.
|
||||
*/
|
||||
private AzureOpenAiChatOptions defaultOptions;
|
||||
|
||||
/**
|
||||
* The {@link OpenAIClient} used to interact with the Azure OpenAI service.
|
||||
*/
|
||||
private final OpenAIClient openAIClient;
|
||||
|
||||
/**
|
||||
* The configuration information for a chat completions request.
|
||||
*/
|
||||
private AzureOpenAiChatOptions defaultOptions;
|
||||
|
||||
public AzureOpenAiChatModel(OpenAIClient microsoftOpenAiClient) {
|
||||
this(microsoftOpenAiClient,
|
||||
AzureOpenAiChatOptions.builder()
|
||||
@@ -277,7 +282,17 @@ public class AzureOpenAiChatModel
|
||||
|
||||
switch (message.getMessageType()) {
|
||||
case USER:
|
||||
return new ChatRequestUserMessage(message.getContent());
|
||||
// https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/openai/azure-ai-openai/README.md#text-completions-with-images
|
||||
List<ChatMessageContentItem> items = new ArrayList<>();
|
||||
items.add(new ChatMessageTextContentItem(message.getContent()));
|
||||
if (!CollectionUtils.isEmpty(message.getMedia())) {
|
||||
items.addAll(message.getMedia()
|
||||
.stream()
|
||||
.map(media -> new ChatMessageImageContentItem(
|
||||
new ChatMessageImageUrl(media.getData().toString())))
|
||||
.toList());
|
||||
}
|
||||
return new ChatRequestUserMessage(items);
|
||||
case SYSTEM:
|
||||
return new ChatRequestSystemMessage(message.getContent());
|
||||
case ASSISTANT:
|
||||
|
||||
@@ -15,23 +15,26 @@
|
||||
*/
|
||||
package org.springframework.ai.azure.openai;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.azure.ai.openai.OpenAIClient;
|
||||
import com.azure.ai.openai.OpenAIClientBuilder;
|
||||
import com.azure.core.credential.AzureKeyCredential;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.chat.prompt.PromptTemplate;
|
||||
import org.springframework.ai.chat.prompt.SystemPromptTemplate;
|
||||
@@ -43,20 +46,22 @@ import org.springframework.boot.SpringBootConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.core.convert.support.DefaultConversionService;
|
||||
import org.springframework.util.MimeTypeUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import com.azure.ai.openai.OpenAIClient;
|
||||
import com.azure.ai.openai.OpenAIClientBuilder;
|
||||
import com.azure.core.credential.AzureKeyCredential;
|
||||
|
||||
@SpringBootTest(classes = AzureOpenAiChatModelIT.TestConfiguration.class)
|
||||
@EnabledIfEnvironmentVariable(named = "AZURE_OPENAI_API_KEY", matches = ".+")
|
||||
@EnabledIfEnvironmentVariable(named = "AZURE_OPENAI_ENDPOINT", matches = ".+")
|
||||
class AzureOpenAiChatModelIT {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(AzureOpenAiChatModelIT.class);
|
||||
|
||||
@Autowired
|
||||
private AzureOpenAiChatModel chatModel;
|
||||
|
||||
record ActorsFilms(String actor, List<String> movies) {
|
||||
}
|
||||
|
||||
@Test
|
||||
void roleTest() {
|
||||
Message systemMessage = new SystemPromptTemplate("""
|
||||
@@ -130,9 +135,6 @@ class AzureOpenAiChatModelIT {
|
||||
assertThat(actorsFilms.actor()).isNotNull();
|
||||
}
|
||||
|
||||
record ActorsFilmsRecord(String actor, List<String> movies) {
|
||||
}
|
||||
|
||||
@Test
|
||||
void beanOutputConverterRecords() {
|
||||
|
||||
@@ -183,6 +185,31 @@ class AzureOpenAiChatModelIT {
|
||||
assertThat(actorsFilms.movies()).hasSize(5);
|
||||
}
|
||||
|
||||
@Test
|
||||
void multiModalityImageUrl() throws IOException {
|
||||
|
||||
// TODO: add url method that wrapps the checked exception.
|
||||
URL url = new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png");
|
||||
|
||||
// @formatter:off
|
||||
String response = ChatClient.create(chatModel).prompt()
|
||||
.options(AzureOpenAiChatOptions.builder().withDeploymentName("gpt-4o").build())
|
||||
.user(u -> u.text("Explain what do you see on this picture?").media(MimeTypeUtils.IMAGE_PNG, url))
|
||||
.call()
|
||||
.content();
|
||||
// @formatter:on
|
||||
|
||||
logger.info(response);
|
||||
assertThat(response).contains("bananas", "apple");
|
||||
assertThat(response).containsAnyOf("bowl", "basket");
|
||||
}
|
||||
|
||||
record ActorsFilms(String actor, List<String> movies) {
|
||||
}
|
||||
|
||||
record ActorsFilmsRecord(String actor, List<String> movies) {
|
||||
}
|
||||
|
||||
@SpringBootConfiguration
|
||||
public static class TestConfiguration {
|
||||
|
||||
|
||||
@@ -9,12 +9,16 @@ import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.ai.azure.openai.AzureOpenAiImageModel;
|
||||
import org.springframework.ai.azure.openai.AzureOpenAiImageOptions;
|
||||
import org.springframework.ai.azure.openai.metadata.AzureOpenAiImageGenerationMetadata;
|
||||
import org.springframework.ai.image.*;
|
||||
import org.springframework.ai.image.Image;
|
||||
import org.springframework.ai.image.ImageModel;
|
||||
import org.springframework.ai.image.ImageOptionsBuilder;
|
||||
import org.springframework.ai.image.ImagePrompt;
|
||||
import org.springframework.ai.image.ImageResponse;
|
||||
import org.springframework.ai.image.ImageResponseMetadata;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringBootConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
@@ -142,6 +142,44 @@ You can register custom Java functions with the AzureOpenAiChatModel and have th
|
||||
This is a powerful technique to connect the LLM capabilities with external tools and APIs.
|
||||
Read more about xref:api/chat/functions/azure-open-ai-chat-functions.adoc[Azure OpenAI Function Calling].
|
||||
|
||||
== Multimodal
|
||||
|
||||
Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats.
|
||||
Presently, the Azure OpenAI `gpt-4o` model offers multimodal support.
|
||||
|
||||
The Azure OpenAI can incorporate a list of base64-encoded images or image urls with the message.
|
||||
Spring AI’s link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java[Message] interface facilitates multimodal AI models by introducing the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java[Media] type.
|
||||
This type encompasses data and details regarding media attachments in messages, utilizing Spring’s `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data.
|
||||
|
||||
Below is a code example excerpted from link:https://github.com/spring-projects/spring-ai/blob/b3cfa2b900ea785e055e4ff71086eeb52f6578a3/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java[OpenAiChatModelIT.java], illustrating the fusion of user text with an image using the the `GPT_4_VISION_PREVIEW` model.
|
||||
|
||||
[source,java]
|
||||
----
|
||||
URL url = new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png");
|
||||
String response = ChatClient.create(chatModel).prompt()
|
||||
.options(AzureOpenAiChatOptions.builder().withDeploymentName("gpt4o").build())
|
||||
.user(u -> u.text("Explain what do you see on this picture?").media(MimeTypeUtils.IMAGE_PNG, url))
|
||||
.call()
|
||||
.content();
|
||||
----
|
||||
|
||||
TIP: you can pass multiple images as well.
|
||||
|
||||
It takes as an input the `multimodal.test.png` image:
|
||||
|
||||
image::multimodal.test.png[Multimodal Test Image, 200, 200, align="left"]
|
||||
|
||||
along with the text message "Explain what do you see on this picture?", and generates a response like this:
|
||||
|
||||
----
|
||||
This is an image of a fruit bowl with a simple design. The bowl is made of metal with curved wire edges that
|
||||
create an open structure, allowing the fruit to be visible from all angles. Inside the bowl, there are two
|
||||
yellow bananas resting on top of what appears to be a red apple. The bananas are slightly overripe, as
|
||||
indicated by the brown spots on their peels. The bowl has a metal ring at the top, likely to serve as a handle
|
||||
for carrying. The bowl is placed on a flat surface with a neutral-colored background that provides a clear
|
||||
view of the fruit inside.
|
||||
----
|
||||
|
||||
== Sample Controller
|
||||
|
||||
https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-azure-openai-spring-boot-starter` to your pom (or gradle) dependencies.
|
||||
|
||||
@@ -73,4 +73,5 @@ Latest version of Spring AI provides multimodal support for the following Chat C
|
||||
* xref:api/chat/ollama-chat.adoc#_multimodal[Ollama - (LlaVa and Baklava models)]
|
||||
* xref:api/chat/vertexai-gemini-chat.adoc#_multimodal[Vertex AI Gemini - (gemini-pro-vision model)]
|
||||
* xref:api/chat/anthropic-chat.adoc#_multimodal[Anthropic Claude 3]
|
||||
* xref:api/chat/bedrock/bedrock-anthropic3.adoc#_multimodal[AWS Bedrock Anthropic Claude 3]
|
||||
* xref:api/chat/bedrock/bedrock-anthropic3.adoc#_multimodal[AWS Bedrock Anthropic Claude 3]
|
||||
* xref:api/chat/azure-openai-chat.adoc#_multimodal[Azure Open AI - (GPT-4o models)]
|
||||
Reference in New Issue
Block a user