Fix Azure chat client image handling for byte arrays

Resolve issue with AzureOpenAiChatModel processing non-public images as
byte arrays. Implement handling for both URL strings and byte arrays,
converting latter to base64 encoded data URLs. Add test case for
resource-based media data. Update documentation with sample for
classpath resources.

models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java

@@ -16,6 +16,7 @@
 package org.springframework.ai.azure.openai;
 
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -41,6 +42,7 @@ import org.springframework.ai.chat.model.ChatResponse;
 import org.springframework.ai.chat.model.Generation;
 import org.springframework.ai.chat.prompt.ChatOptions;
 import org.springframework.ai.chat.prompt.Prompt;
+import org.springframework.ai.model.Media;
 import org.springframework.ai.model.ModelOptionsUtils;
 import org.springframework.ai.model.function.FunctionCallback;
 import org.springframework.ai.model.function.FunctionCallbackContext;
@@ -322,8 +324,7 @@ public class AzureOpenAiChatModel extends AbstractToolCallSupport implements Cha
 					if (!CollectionUtils.isEmpty(userMessage.getMedia())) {
 						items.addAll(userMessage.getMedia()
 							.stream()
-							.map(media -> new ChatMessageImageContentItem(
-									new ChatMessageImageUrl(media.getData().toString())))
+							.map(media -> new ChatMessageImageContentItem(new ChatMessageImageUrl(getMediaUrl(media))))
 							.toList());
 					}
 				}
@@ -364,6 +365,18 @@ public class AzureOpenAiChatModel extends AbstractToolCallSupport implements Cha
 		}
 	}
 
+	private String getMediaUrl(Media media) {
+		Object data = media.getData();
+		if (data instanceof String dataUrl)
+			return dataUrl;
+		else if (data instanceof byte[] dataBytes) {
+			String base64EncodedData = Base64.getEncoder().encodeToString(dataBytes);
+			return "data:" + media.getMimeType() + ";base64," + base64EncodedData;
+		}
+		else
+			throw new IllegalArgumentException("Unknown media data type " + data.getClass().getName());
+	}
+
 	private ChatGenerationMetadata generateChoiceMetadata(ChatChoice choice) {
 		return ChatGenerationMetadata.from(String.valueOf(choice.getFinishReason()), choice.getContentFilterResults());
 	}

models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java

@@ -41,6 +41,8 @@ import org.springframework.boot.SpringBootConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.context.annotation.Bean;
 import org.springframework.core.convert.support.DefaultConversionService;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
 import org.springframework.util.MimeTypeUtils;
 
 import java.io.IOException;
@@ -206,6 +208,24 @@ class AzureOpenAiChatModelIT {
 		assertThat(response).containsAnyOf("bowl", "basket");
 	}
 
+	@Test
+	void multiModalityImageResource() {
+
+		Resource resource = new ClassPathResource("multimodality/multimodal.test.png");
+
+		// @formatter:off
+		String response = ChatClient.create(chatModel).prompt()
+				.options(AzureOpenAiChatOptions.builder().withDeploymentName("gpt-4o").build())
+				.user(u -> u.text("Explain what do you see on this picture?").media(MimeTypeUtils.IMAGE_PNG, resource))
+				.call()
+				.content();
+		// @formatter:on
+
+		logger.info(response);
+		assertThat(response).contains("bananas", "apple");
+		assertThat(response).containsAnyOf("bowl", "basket");
+	}
+
 	record ActorsFilms(String actor, List<String> movies) {
 	}
 

spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc

@@ -215,6 +215,21 @@ for carrying. The bowl is placed on a flat surface with a neutral-colored backgr
 view of the fruit inside.
 ----
 
+You can also pass in a classpath resource instead of a URL as shown in the example below
+
+[source,java]
+----
+Resource resource = new ClassPathResource("multimodality/multimodal.test.png");
+
+String response = ChatClient.create(chatModel).prompt()
+    .options(AzureOpenAiChatOptions.builder()
+    .withDeploymentName("gpt-4o").build())
+    .user(u -> u.text("Explain what do you see on this picture?")
+    .media(MimeTypeUtils.IMAGE_PNG, resource))
+    .call()
+    .content();
+----
+
 == Sample Controller
 
 https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-azure-openai-spring-boot-starter` to your pom (or gradle) dependencies.