From c6ccc1a4a3d09f027199fe6bc6c183d907adf24b Mon Sep 17 00:00:00 2001 From: jonghoonpark Date: Mon, 21 Apr 2025 17:38:09 +0900 Subject: [PATCH] Make allow image url for Anthropic API Signed-off-by: jonghoonpark --- .../ai/anthropic/AnthropicChatModel.java | 16 ++++++++++++++-- .../ai/anthropic/api/AnthropicApi.java | 9 +++++++-- .../anthropic/client/AnthropicChatClientIT.java | 8 +++----- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java b/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java index f36a0eb11..3a43ca404 100644 --- a/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java +++ b/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java @@ -88,6 +88,7 @@ import org.springframework.util.StringUtils; * @author Thomas Vitale * @author Claudio Silva Junior * @author Alexandros Pappas + * @author Jonghoon Park * @since 1.0.0 */ public class AnthropicChatModel implements ChatModel { @@ -355,6 +356,18 @@ public class AnthropicChatModel implements ChatModel { .build(); } + private Source getSourceByMedia(Media media) { + String data = this.fromMediaData(media.getData()); + + // http is not allowed and redirect not allowed + if (data.startsWith("https://")) { + return new Source(data); + } + else { + return new Source(media.getMimeType().toString(), data); + } + } + private String fromMediaData(Object mediaData) { if (mediaData instanceof byte[] bytes) { return Base64.getEncoder().encodeToString(bytes); @@ -455,8 +468,7 @@ public class AnthropicChatModel implements ChatModel { if (!CollectionUtils.isEmpty(userMessage.getMedia())) { List mediaContent = userMessage.getMedia().stream().map(media -> { Type contentBlockType = getContentBlockTypeByMedia(media); - var source = new Source(media.getMimeType().toString(), - this.fromMediaData(media.getData())); + var source = getSourceByMedia(media); return new ContentBlock(contentBlockType, source); }).toList(); contents.addAll(mediaContent); diff --git a/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/api/AnthropicApi.java b/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/api/AnthropicApi.java index a4105989b..d9431a372 100644 --- a/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/api/AnthropicApi.java +++ b/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/api/AnthropicApi.java @@ -933,7 +933,8 @@ public class AnthropicApi { // @formatter:off @JsonProperty("type") String type, @JsonProperty("media_type") String mediaType, - @JsonProperty("data") String data) { + @JsonProperty("data") String data, + @JsonProperty("url") String url) { // @formatter:on /** @@ -942,7 +943,11 @@ public class AnthropicApi { * @param data The content data. */ public Source(String mediaType, String data) { - this("base64", mediaType, data); + this("base64", mediaType, data, null); + } + + public Source(String url) { + this("url", null, null, url); } } diff --git a/models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/client/AnthropicChatClientIT.java b/models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/client/AnthropicChatClientIT.java index 556cd548c..5353d48c4 100644 --- a/models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/client/AnthropicChatClientIT.java +++ b/models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/client/AnthropicChatClientIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -301,14 +301,12 @@ class AnthropicChatClientIT { assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand"); } - @Disabled("Currently Anthropic API does not support external image URLs") @ParameterizedTest(name = "{0} : {displayName} ") - @ValueSource(strings = { "claude-3-opus-latest", "claude-3-5-sonnet-latest", "claude-3-haiku-latest", - "claude-3-7-sonnet-latest" }) + @ValueSource(strings = { "claude-3-opus-latest", "claude-3-5-sonnet-latest", "claude-3-7-sonnet-latest" }) void multiModalityImageUrl(String modelName) throws IOException { // TODO: add url method that wrapps the checked exception. - URL url = new URL("https://docs.spring.io/spring-ai/reference/1.0.0-SNAPSHOT/_images/multimodal.test.png"); + URL url = new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png"); // @formatter:off String response = ChatClient.create(this.chatModel).prompt()