diff --git a/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java index f70217e90..ec3872e00 100644 --- a/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java +++ b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java @@ -16,6 +16,7 @@ package org.springframework.ai.azure.openai; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -41,6 +42,7 @@ import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.model.Media; import org.springframework.ai.model.ModelOptionsUtils; import org.springframework.ai.model.function.FunctionCallback; import org.springframework.ai.model.function.FunctionCallbackContext; @@ -322,8 +324,7 @@ public class AzureOpenAiChatModel extends AbstractToolCallSupport implements Cha if (!CollectionUtils.isEmpty(userMessage.getMedia())) { items.addAll(userMessage.getMedia() .stream() - .map(media -> new ChatMessageImageContentItem( - new ChatMessageImageUrl(media.getData().toString()))) + .map(media -> new ChatMessageImageContentItem(new ChatMessageImageUrl(getMediaUrl(media)))) .toList()); } } @@ -364,6 +365,18 @@ public class AzureOpenAiChatModel extends AbstractToolCallSupport implements Cha } } + private String getMediaUrl(Media media) { + Object data = media.getData(); + if (data instanceof String dataUrl) + return dataUrl; + else if (data instanceof byte[] dataBytes) { + String base64EncodedData = Base64.getEncoder().encodeToString(dataBytes); + return "data:" + media.getMimeType() + ";base64," + base64EncodedData; + } + else + throw new IllegalArgumentException("Unknown media data type " + data.getClass().getName()); + } + private ChatGenerationMetadata generateChoiceMetadata(ChatChoice choice) { return ChatGenerationMetadata.from(String.valueOf(choice.getFinishReason()), choice.getContentFilterResults()); } diff --git a/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java b/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java index cc0cba054..957c8b9ce 100644 --- a/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java +++ b/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java @@ -41,6 +41,8 @@ import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.Bean; import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; import org.springframework.util.MimeTypeUtils; import java.io.IOException; @@ -206,6 +208,24 @@ class AzureOpenAiChatModelIT { assertThat(response).containsAnyOf("bowl", "basket"); } + @Test + void multiModalityImageResource() { + + Resource resource = new ClassPathResource("multimodality/multimodal.test.png"); + + // @formatter:off + String response = ChatClient.create(chatModel).prompt() + .options(AzureOpenAiChatOptions.builder().withDeploymentName("gpt-4o").build()) + .user(u -> u.text("Explain what do you see on this picture?").media(MimeTypeUtils.IMAGE_PNG, resource)) + .call() + .content(); + // @formatter:on + + logger.info(response); + assertThat(response).contains("bananas", "apple"); + assertThat(response).containsAnyOf("bowl", "basket"); + } + record ActorsFilms(String actor, List movies) { } diff --git a/models/spring-ai-azure-openai/src/test/resources/multimodality/multimodal.test.png b/models/spring-ai-azure-openai/src/test/resources/multimodality/multimodal.test.png new file mode 100644 index 000000000..4f8984541 Binary files /dev/null and b/models/spring-ai-azure-openai/src/test/resources/multimodality/multimodal.test.png differ diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc index e856fc3a4..7b43c4768 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc @@ -215,6 +215,21 @@ for carrying. The bowl is placed on a flat surface with a neutral-colored backgr view of the fruit inside. ---- +You can also pass in a classpath resource instead of a URL as shown in the example below + +[source,java] +---- +Resource resource = new ClassPathResource("multimodality/multimodal.test.png"); + +String response = ChatClient.create(chatModel).prompt() + .options(AzureOpenAiChatOptions.builder() + .withDeploymentName("gpt-4o").build()) + .user(u -> u.text("Explain what do you see on this picture?") + .media(MimeTypeUtils.IMAGE_PNG, resource)) + .call() + .content(); +---- + == Sample Controller https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-azure-openai-spring-boot-starter` to your pom (or gradle) dependencies.