Update AnthropicChatModel and Spring AI Documentation for Multimodal Support
- Enhance AnthropicChatModel to support PDF and document content types - Introduce getContentBlockTypeByMedia method for flexible media type handling - Update ContentBlock handling to dynamically determine content type for media - Add multimodal PDF support test case for Claude 3.5 Sonnet - Update documentation to reflect PDF and multimodal capabilities - Modify comparison chart to show PDF support for Anthropic Claude Fixese #1819 review
This commit is contained in:
Christian Tzolov
committed by
Mark Pollack
Mark Pollack
parent
9d207e37be
commit
e543cd082d
@@ -37,6 +37,7 @@ import org.springframework.ai.anthropic.api.AnthropicApi.AnthropicMessage;
|
||||
import org.springframework.ai.anthropic.api.AnthropicApi.ChatCompletionRequest;
|
||||
import org.springframework.ai.anthropic.api.AnthropicApi.ChatCompletionResponse;
|
||||
import org.springframework.ai.anthropic.api.AnthropicApi.ContentBlock;
|
||||
import org.springframework.ai.anthropic.api.AnthropicApi.ContentBlock.Source;
|
||||
import org.springframework.ai.anthropic.api.AnthropicApi.ContentBlock.Type;
|
||||
import org.springframework.ai.anthropic.api.AnthropicApi.Role;
|
||||
import org.springframework.ai.anthropic.metadata.AnthropicUsage;
|
||||
@@ -58,6 +59,7 @@ import org.springframework.ai.chat.observation.DefaultChatModelObservationConven
|
||||
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||
import org.springframework.ai.chat.prompt.ChatOptionsBuilder;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.model.Media;
|
||||
import org.springframework.ai.model.ModelOptionsUtils;
|
||||
import org.springframework.ai.model.function.FunctionCallback;
|
||||
import org.springframework.ai.model.function.FunctionCallbackContext;
|
||||
@@ -355,6 +357,18 @@ public class AnthropicChatModel extends AbstractToolCallSupport implements ChatM
|
||||
|
||||
}
|
||||
|
||||
private Type getContentBlockTypeByMedia(Media media) {
|
||||
String mimeType = media.getMimeType().toString();
|
||||
if (mimeType.startsWith("image")) {
|
||||
return Type.IMAGE;
|
||||
}
|
||||
else if (mimeType.contains("pdf")) {
|
||||
return Type.DOCUMENT;
|
||||
}
|
||||
throw new IllegalArgumentException("Unsupported media type: " + mimeType
|
||||
+ ". Supported types are: images (image/*) and PDF documents (application/pdf)");
|
||||
}
|
||||
|
||||
ChatCompletionRequest createRequest(Prompt prompt, boolean stream) {
|
||||
|
||||
Set<String> functionsForThisRequest = new HashSet<>();
|
||||
@@ -367,11 +381,12 @@ public class AnthropicChatModel extends AbstractToolCallSupport implements ChatM
|
||||
List<ContentBlock> contents = new ArrayList<>(List.of(new ContentBlock(message.getContent())));
|
||||
if (message instanceof UserMessage userMessage) {
|
||||
if (!CollectionUtils.isEmpty(userMessage.getMedia())) {
|
||||
List<ContentBlock> mediaContent = userMessage.getMedia()
|
||||
.stream()
|
||||
.map(media -> new ContentBlock(media.getMimeType().toString(),
|
||||
this.fromMediaData(media.getData())))
|
||||
.toList();
|
||||
List<ContentBlock> mediaContent = userMessage.getMedia().stream().map(media -> {
|
||||
Type contentBlockType = getContentBlockTypeByMedia(media);
|
||||
var source = new Source(media.getMimeType().toString(),
|
||||
this.fromMediaData(media.getData()));
|
||||
return new ContentBlock(contentBlockType, source);
|
||||
}).toList();
|
||||
contents.addAll(mediaContent);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ public class AnthropicApi {
|
||||
|
||||
public static final String DEFAULT_ANTHROPIC_VERSION = "2023-06-01";
|
||||
|
||||
public static final String DEFAULT_ANTHROPIC_BETA_VERSION = "tools-2024-04-04";
|
||||
public static final String DEFAULT_ANTHROPIC_BETA_VERSION = "tools-2024-04-04,pdfs-2024-09-25";
|
||||
|
||||
public static final String BETA_MAX_TOKENS = "max-tokens-3-5-sonnet-2024-07-15";
|
||||
|
||||
@@ -230,18 +230,23 @@ public class AnthropicApi {
|
||||
/**
|
||||
* The claude-3-5-sonnet-20241022 model.
|
||||
*/
|
||||
CLAUDE_3_5_SONNET("claude-3-5-sonnet-20241022"),
|
||||
CLAUDE_3_5_SONNET("claude-3-5-sonnet-latest"),
|
||||
|
||||
/**
|
||||
* The CLAUDE_3_OPUS
|
||||
*/
|
||||
CLAUDE_3_OPUS("claude-3-opus-20240229"),
|
||||
CLAUDE_3_OPUS("claude-3-opus-latest"),
|
||||
|
||||
/**
|
||||
* The CLAUDE_3_SONNET
|
||||
*/
|
||||
CLAUDE_3_SONNET("claude-3-sonnet-20240229"),
|
||||
|
||||
/**
|
||||
* The CLAUDE 3.5 HAIKU
|
||||
*/
|
||||
CLAUDE_3_5_HAIKU("claude-3-5-haiku-latest"),
|
||||
|
||||
/**
|
||||
* The CLAUDE_3_HAIKU
|
||||
*/
|
||||
@@ -296,18 +301,18 @@ public class AnthropicApi {
|
||||
public enum Role {
|
||||
|
||||
// @formatter:off
|
||||
/**
|
||||
* The user role.
|
||||
*/
|
||||
@JsonProperty("user")
|
||||
USER,
|
||||
/**
|
||||
* The user role.
|
||||
*/
|
||||
@JsonProperty("user")
|
||||
USER,
|
||||
|
||||
/**
|
||||
* The assistant role.
|
||||
*/
|
||||
@JsonProperty("assistant")
|
||||
ASSISTANT
|
||||
// @formatter:on
|
||||
/**
|
||||
* The assistant role.
|
||||
*/
|
||||
@JsonProperty("assistant")
|
||||
ASSISTANT
|
||||
// @formatter:on
|
||||
|
||||
}
|
||||
|
||||
@@ -436,18 +441,18 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ChatCompletionRequest(
|
||||
// @formatter:off
|
||||
@JsonProperty("model") String model,
|
||||
@JsonProperty("messages") List<AnthropicMessage> messages,
|
||||
@JsonProperty("system") String system,
|
||||
@JsonProperty("max_tokens") Integer maxTokens,
|
||||
@JsonProperty("metadata") Metadata metadata,
|
||||
@JsonProperty("stop_sequences") List<String> stopSequences,
|
||||
@JsonProperty("stream") Boolean stream,
|
||||
@JsonProperty("temperature") Double temperature,
|
||||
@JsonProperty("top_p") Double topP,
|
||||
@JsonProperty("top_k") Integer topK,
|
||||
@JsonProperty("tools") List<Tool> tools) {
|
||||
// @formatter:on
|
||||
@JsonProperty("model") String model,
|
||||
@JsonProperty("messages") List<AnthropicMessage> messages,
|
||||
@JsonProperty("system") String system,
|
||||
@JsonProperty("max_tokens") Integer maxTokens,
|
||||
@JsonProperty("metadata") Metadata metadata,
|
||||
@JsonProperty("stop_sequences") List<String> stopSequences,
|
||||
@JsonProperty("stream") Boolean stream,
|
||||
@JsonProperty("temperature") Double temperature,
|
||||
@JsonProperty("top_p") Double topP,
|
||||
@JsonProperty("top_k") Integer topK,
|
||||
@JsonProperty("tools") List<Tool> tools) {
|
||||
// @formatter:on
|
||||
|
||||
public ChatCompletionRequest(String model, List<AnthropicMessage> messages, String system, Integer maxTokens,
|
||||
Double temperature, Boolean stream) {
|
||||
@@ -615,9 +620,9 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record AnthropicMessage(
|
||||
// @formatter:off
|
||||
@JsonProperty("content") List<ContentBlock> content,
|
||||
@JsonProperty("role") Role role) {
|
||||
// @formatter:on
|
||||
@JsonProperty("content") List<ContentBlock> content,
|
||||
@JsonProperty("role") Role role) {
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -639,23 +644,23 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlock(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") Type type,
|
||||
@JsonProperty("source") Source source,
|
||||
@JsonProperty("text") String text,
|
||||
@JsonProperty("type") Type type,
|
||||
@JsonProperty("source") Source source,
|
||||
@JsonProperty("text") String text,
|
||||
|
||||
// applicable only for streaming responses.
|
||||
@JsonProperty("index") Integer index,
|
||||
// applicable only for streaming responses.
|
||||
@JsonProperty("index") Integer index,
|
||||
|
||||
// tool_use response only
|
||||
@JsonProperty("id") String id,
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("input") Map<String, Object> input,
|
||||
// tool_use response only
|
||||
@JsonProperty("id") String id,
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("input") Map<String, Object> input,
|
||||
|
||||
// tool_result response only
|
||||
@JsonProperty("tool_use_id") String toolUseId,
|
||||
@JsonProperty("content") String content
|
||||
) {
|
||||
// @formatter:on
|
||||
// tool_result response only
|
||||
@JsonProperty("tool_use_id") String toolUseId,
|
||||
@JsonProperty("content") String content
|
||||
) {
|
||||
// @formatter:on
|
||||
|
||||
/**
|
||||
* Create content block
|
||||
@@ -666,6 +671,15 @@ public class AnthropicApi {
|
||||
this(new Source(mediaType, data));
|
||||
}
|
||||
|
||||
/**
|
||||
* Create content block
|
||||
* @param type The type of the content.
|
||||
* @param source The source of the content.
|
||||
*/
|
||||
public ContentBlock(Type type, Source source) {
|
||||
this(type, source, null, null, null, null, null, null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create content block
|
||||
* @param source The source of the content.
|
||||
@@ -755,7 +769,13 @@ public class AnthropicApi {
|
||||
* Image message.
|
||||
*/
|
||||
@JsonProperty("image")
|
||||
IMAGE("image");
|
||||
IMAGE("image"),
|
||||
|
||||
/**
|
||||
* Document message.
|
||||
*/
|
||||
@JsonProperty("document")
|
||||
DOCUMENT("document");
|
||||
|
||||
public final String value;
|
||||
|
||||
@@ -785,10 +805,10 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record Source(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("media_type") String mediaType,
|
||||
@JsonProperty("data") String data) {
|
||||
// @formatter:on
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("media_type") String mediaType,
|
||||
@JsonProperty("data") String data) {
|
||||
// @formatter:on
|
||||
|
||||
/**
|
||||
* Create source
|
||||
@@ -817,10 +837,10 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record Tool(
|
||||
// @formatter:off
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("description") String description,
|
||||
@JsonProperty("input_schema") Map<String, Object> inputSchema) {
|
||||
// @formatter:on
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("description") String description,
|
||||
@JsonProperty("input_schema") Map<String, Object> inputSchema) {
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
// CB START EVENT
|
||||
@@ -843,15 +863,15 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ChatCompletionResponse(
|
||||
// @formatter:off
|
||||
@JsonProperty("id") String id,
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("role") Role role,
|
||||
@JsonProperty("content") List<ContentBlock> content,
|
||||
@JsonProperty("model") String model,
|
||||
@JsonProperty("stop_reason") String stopReason,
|
||||
@JsonProperty("stop_sequence") String stopSequence,
|
||||
@JsonProperty("usage") Usage usage) {
|
||||
// @formatter:on
|
||||
@JsonProperty("id") String id,
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("role") Role role,
|
||||
@JsonProperty("content") List<ContentBlock> content,
|
||||
@JsonProperty("model") String model,
|
||||
@JsonProperty("stop_reason") String stopReason,
|
||||
@JsonProperty("stop_sequence") String stopSequence,
|
||||
@JsonProperty("usage") Usage usage) {
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
// CB DELTA EVENT
|
||||
@@ -865,143 +885,143 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record Usage(
|
||||
// @formatter:off
|
||||
@JsonProperty("input_tokens") Integer inputTokens,
|
||||
@JsonProperty("output_tokens") Integer outputTokens) {
|
||||
// @formatter:off
|
||||
}
|
||||
@JsonProperty("input_tokens") Integer inputTokens,
|
||||
@JsonProperty("output_tokens") Integer outputTokens) {
|
||||
// @formatter:off
|
||||
}
|
||||
|
||||
/// ECB STOP
|
||||
/// ECB STOP
|
||||
|
||||
/**
|
||||
* Special event used to aggregate multiple tool use events into a single event with
|
||||
* list of aggregated ContentBlockToolUse.
|
||||
*/
|
||||
public static class ToolUseAggregationEvent implements StreamEvent {
|
||||
/**
|
||||
* Special event used to aggregate multiple tool use events into a single event with
|
||||
* list of aggregated ContentBlockToolUse.
|
||||
*/
|
||||
public static class ToolUseAggregationEvent implements StreamEvent {
|
||||
|
||||
private Integer index;
|
||||
private Integer index;
|
||||
|
||||
private String id;
|
||||
private String id;
|
||||
|
||||
private String name;
|
||||
private String name;
|
||||
|
||||
private String partialJson = "";
|
||||
private String partialJson = "";
|
||||
|
||||
private List<ContentBlockStartEvent.ContentBlockToolUse> toolContentBlocks = new ArrayList<>();
|
||||
private List<ContentBlockStartEvent.ContentBlockToolUse> toolContentBlocks = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public EventType type() {
|
||||
return EventType.TOOL_USE_AGGREGATE;
|
||||
}
|
||||
@Override
|
||||
public EventType type() {
|
||||
return EventType.TOOL_USE_AGGREGATE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get tool content blocks.
|
||||
* @return The tool content blocks.
|
||||
*/
|
||||
public List<ContentBlockStartEvent.ContentBlockToolUse> getToolContentBlocks() {
|
||||
return this.toolContentBlocks;
|
||||
}
|
||||
/**
|
||||
* Get tool content blocks.
|
||||
* @return The tool content blocks.
|
||||
*/
|
||||
public List<ContentBlockStartEvent.ContentBlockToolUse> getToolContentBlocks() {
|
||||
return this.toolContentBlocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the event is empty.
|
||||
* @return True if the event is empty, false otherwise.
|
||||
*/
|
||||
public boolean isEmpty() {
|
||||
return (this.index == null || this.id == null || this.name == null
|
||||
|| !StringUtils.hasText(this.partialJson));
|
||||
}
|
||||
/**
|
||||
* Check if the event is empty.
|
||||
* @return True if the event is empty, false otherwise.
|
||||
*/
|
||||
public boolean isEmpty() {
|
||||
return (this.index == null || this.id == null || this.name == null
|
||||
|| !StringUtils.hasText(this.partialJson));
|
||||
}
|
||||
|
||||
ToolUseAggregationEvent withIndex(Integer index) {
|
||||
this.index = index;
|
||||
return this;
|
||||
}
|
||||
ToolUseAggregationEvent withIndex(Integer index) {
|
||||
this.index = index;
|
||||
return this;
|
||||
}
|
||||
|
||||
ToolUseAggregationEvent withId(String id) {
|
||||
this.id = id;
|
||||
return this;
|
||||
}
|
||||
ToolUseAggregationEvent withId(String id) {
|
||||
this.id = id;
|
||||
return this;
|
||||
}
|
||||
|
||||
ToolUseAggregationEvent withName(String name) {
|
||||
this.name = name;
|
||||
return this;
|
||||
}
|
||||
ToolUseAggregationEvent withName(String name) {
|
||||
this.name = name;
|
||||
return this;
|
||||
}
|
||||
|
||||
ToolUseAggregationEvent appendPartialJson(String partialJson) {
|
||||
this.partialJson = this.partialJson + partialJson;
|
||||
return this;
|
||||
}
|
||||
ToolUseAggregationEvent appendPartialJson(String partialJson) {
|
||||
this.partialJson = this.partialJson + partialJson;
|
||||
return this;
|
||||
}
|
||||
|
||||
void squashIntoContentBlock() {
|
||||
Map<String, Object> map = (StringUtils.hasText(this.partialJson))
|
||||
? ModelOptionsUtils.jsonToMap(this.partialJson) : Map.of();
|
||||
this.toolContentBlocks.add(new ContentBlockStartEvent.ContentBlockToolUse("tool_use", this.id, this.name, map));
|
||||
this.index = null;
|
||||
this.id = null;
|
||||
this.name = null;
|
||||
this.partialJson = "";
|
||||
}
|
||||
void squashIntoContentBlock() {
|
||||
Map<String, Object> map = (StringUtils.hasText(this.partialJson))
|
||||
? ModelOptionsUtils.jsonToMap(this.partialJson) : Map.of();
|
||||
this.toolContentBlocks.add(new ContentBlockStartEvent.ContentBlockToolUse("tool_use", this.id, this.name, map));
|
||||
this.index = null;
|
||||
this.id = null;
|
||||
this.name = null;
|
||||
this.partialJson = "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "EventToolUseBuilder [index=" + this.index + ", id=" + this.id + ", name=" + this.name + ", partialJson="
|
||||
+ this.partialJson + ", toolUseMap=" + this.toolContentBlocks + "]";
|
||||
}
|
||||
@Override
|
||||
public String toString() {
|
||||
return "EventToolUseBuilder [index=" + this.index + ", id=" + this.id + ", name=" + this.name + ", partialJson="
|
||||
+ this.partialJson + ", toolUseMap=" + this.toolContentBlocks + "]";
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////
|
||||
/// MESSAGE EVENTS
|
||||
///////////////////////////////////////
|
||||
///////////////////////////////////////
|
||||
/// MESSAGE EVENTS
|
||||
///////////////////////////////////////
|
||||
|
||||
// MESSAGE START EVENT
|
||||
// MESSAGE START EVENT
|
||||
|
||||
/**
|
||||
* Content block start event.
|
||||
* @param type The event type.
|
||||
* @param index The index of the content block.
|
||||
* @param contentBlock The content block body.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockStartEvent(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("index") Integer index,
|
||||
@JsonProperty("content_block") ContentBlockBody contentBlock) implements StreamEvent {
|
||||
/**
|
||||
* Content block start event.
|
||||
* @param type The event type.
|
||||
* @param index The index of the content block.
|
||||
* @param contentBlock The content block body.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockStartEvent(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("index") Integer index,
|
||||
@JsonProperty("content_block") ContentBlockBody contentBlock) implements StreamEvent {
|
||||
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type",
|
||||
visible = true)
|
||||
@JsonSubTypes({ @JsonSubTypes.Type(value = ContentBlockToolUse.class, name = "tool_use"),
|
||||
@JsonSubTypes.Type(value = ContentBlockText.class, name = "text") })
|
||||
public interface ContentBlockBody {
|
||||
String type();
|
||||
}
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type",
|
||||
visible = true)
|
||||
@JsonSubTypes({ @JsonSubTypes.Type(value = ContentBlockToolUse.class, name = "tool_use"),
|
||||
@JsonSubTypes.Type(value = ContentBlockText.class, name = "text") })
|
||||
public interface ContentBlockBody {
|
||||
String type();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tool use content block.
|
||||
* @param type The content block type.
|
||||
* @param id The tool use id.
|
||||
* @param name The tool use name.
|
||||
* @param input The tool use input.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockToolUse(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("id") String id,
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("input") Map<String, Object> input) implements ContentBlockBody {
|
||||
}
|
||||
/**
|
||||
* Tool use content block.
|
||||
* @param type The content block type.
|
||||
* @param id The tool use id.
|
||||
* @param name The tool use name.
|
||||
* @param input The tool use input.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockToolUse(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("id") String id,
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("input") Map<String, Object> input) implements ContentBlockBody {
|
||||
}
|
||||
|
||||
/**
|
||||
* Text content block.
|
||||
* @param type The content block type.
|
||||
* @param text The text content.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockText(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("text") String text) implements ContentBlockBody {
|
||||
}
|
||||
}
|
||||
// @formatter:on
|
||||
/**
|
||||
* Text content block.
|
||||
* @param type The content block type.
|
||||
* @param text The text content.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockText(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("text") String text) implements ContentBlockBody {
|
||||
}
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
// MESSAGE DELTA EVENT
|
||||
|
||||
@@ -1015,41 +1035,41 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockDeltaEvent(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("index") Integer index,
|
||||
@JsonProperty("delta") ContentBlockDeltaBody delta) implements StreamEvent {
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("index") Integer index,
|
||||
@JsonProperty("delta") ContentBlockDeltaBody delta) implements StreamEvent {
|
||||
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type",
|
||||
visible = true)
|
||||
@JsonSubTypes({ @JsonSubTypes.Type(value = ContentBlockDeltaText.class, name = "text_delta"),
|
||||
@JsonSubTypes.Type(value = ContentBlockDeltaJson.class, name = "input_json_delta") })
|
||||
public interface ContentBlockDeltaBody {
|
||||
String type();
|
||||
}
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type",
|
||||
visible = true)
|
||||
@JsonSubTypes({ @JsonSubTypes.Type(value = ContentBlockDeltaText.class, name = "text_delta"),
|
||||
@JsonSubTypes.Type(value = ContentBlockDeltaJson.class, name = "input_json_delta") })
|
||||
public interface ContentBlockDeltaBody {
|
||||
String type();
|
||||
}
|
||||
|
||||
/**
|
||||
* Text content block delta.
|
||||
* @param type The content block type.
|
||||
* @param text The text content.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockDeltaText(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("text") String text) implements ContentBlockDeltaBody {
|
||||
}
|
||||
/**
|
||||
* Text content block delta.
|
||||
* @param type The content block type.
|
||||
* @param text The text content.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockDeltaText(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("text") String text) implements ContentBlockDeltaBody {
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON content block delta.
|
||||
* @param type The content block type.
|
||||
* @param partialJson The partial JSON content.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockDeltaJson(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("partial_json") String partialJson) implements ContentBlockDeltaBody {
|
||||
}
|
||||
}
|
||||
// @formatter:on
|
||||
/**
|
||||
* JSON content block delta.
|
||||
* @param type The content block type.
|
||||
* @param partialJson The partial JSON content.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockDeltaJson(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("partial_json") String partialJson) implements ContentBlockDeltaBody {
|
||||
}
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
// MESSAGE STOP EVENT
|
||||
|
||||
@@ -1062,10 +1082,10 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ContentBlockStopEvent(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("index") Integer index) implements StreamEvent {
|
||||
}
|
||||
// @formatter:on
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("index") Integer index) implements StreamEvent {
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
/**
|
||||
* Message start event.
|
||||
@@ -1075,10 +1095,10 @@ public class AnthropicApi {
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record MessageStartEvent(// @formatter:off
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("message") ChatCompletionResponse message) implements StreamEvent {
|
||||
}
|
||||
// @formatter:on
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("message") ChatCompletionResponse message) implements StreamEvent {
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
/**
|
||||
* Message delta event.
|
||||
@@ -1090,31 +1110,31 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record MessageDeltaEvent(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("delta") MessageDelta delta,
|
||||
@JsonProperty("usage") MessageDeltaUsage usage) implements StreamEvent {
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("delta") MessageDelta delta,
|
||||
@JsonProperty("usage") MessageDeltaUsage usage) implements StreamEvent {
|
||||
|
||||
/**
|
||||
* Message delta.
|
||||
* @param stopReason The stop reason.
|
||||
* @param stopSequence The stop sequence.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record MessageDelta(
|
||||
@JsonProperty("stop_reason") String stopReason,
|
||||
@JsonProperty("stop_sequence") String stopSequence) {
|
||||
}
|
||||
/**
|
||||
* Message delta.
|
||||
* @param stopReason The stop reason.
|
||||
* @param stopSequence The stop sequence.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record MessageDelta(
|
||||
@JsonProperty("stop_reason") String stopReason,
|
||||
@JsonProperty("stop_sequence") String stopSequence) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Message delta usage.
|
||||
* @param outputTokens The output tokens.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record MessageDeltaUsage(
|
||||
@JsonProperty("output_tokens") Integer outputTokens) {
|
||||
}
|
||||
}
|
||||
// @formatter:on
|
||||
/**
|
||||
* Message delta usage.
|
||||
* @param outputTokens The output tokens.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record MessageDeltaUsage(
|
||||
@JsonProperty("output_tokens") Integer outputTokens) {
|
||||
}
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
/**
|
||||
* Message stop event.
|
||||
@@ -1124,9 +1144,9 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record MessageStopEvent(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") EventType type) implements StreamEvent {
|
||||
}
|
||||
// @formatter:on
|
||||
@JsonProperty("type") EventType type) implements StreamEvent {
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
///////////////////////////////////////
|
||||
/// ERROR EVENT
|
||||
@@ -1140,21 +1160,21 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record ErrorEvent(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("error") Error error) implements StreamEvent {
|
||||
@JsonProperty("type") EventType type,
|
||||
@JsonProperty("error") Error error) implements StreamEvent {
|
||||
|
||||
/**
|
||||
* Error body.
|
||||
* @param type The error type.
|
||||
* @param message The error message.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record Error(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("message") String message) {
|
||||
}
|
||||
}
|
||||
// @formatter:on
|
||||
/**
|
||||
* Error body.
|
||||
* @param type The error type.
|
||||
* @param message The error message.
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record Error(
|
||||
@JsonProperty("type") String type,
|
||||
@JsonProperty("message") String message) {
|
||||
}
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
///////////////////////////////////////
|
||||
/// PING EVENT
|
||||
@@ -1167,8 +1187,8 @@ public class AnthropicApi {
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public record PingEvent(
|
||||
// @formatter:off
|
||||
@JsonProperty("type") EventType type) implements StreamEvent {
|
||||
}
|
||||
// @formatter:on
|
||||
@JsonProperty("type") EventType type) implements StreamEvent {
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,7 @@ import org.springframework.ai.converter.ListOutputConverter;
|
||||
import org.springframework.ai.converter.MapOutputConverter;
|
||||
import org.springframework.ai.model.Media;
|
||||
import org.springframework.ai.model.function.FunctionCallback;
|
||||
import org.springframework.ai.model.function.FunctionCallingOptionsBuilder.PortableFunctionCallingOptions;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.SpringBootConfiguration;
|
||||
@@ -57,6 +58,7 @@ import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.core.convert.support.DefaultConversionService;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.util.MimeType;
|
||||
import org.springframework.util.MimeTypeUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
@@ -246,6 +248,23 @@ class AnthropicChatModelIT {
|
||||
assertThat(response.getResult().getOutput().getContent()).contains("banan", "apple", "basket");
|
||||
}
|
||||
|
||||
@Test
|
||||
void multiModalityPdfTest() throws IOException {
|
||||
|
||||
var pdfData = new ClassPathResource("/spring-ai-reference-overview.pdf");
|
||||
|
||||
var userMessage = new UserMessage(
|
||||
"You are a very professional document summarization specialist. Please summarize the given document.",
|
||||
List.of(new Media(new MimeType("application", "pdf"), pdfData)));
|
||||
|
||||
var response = this.chatModel.call(new Prompt(List.of(userMessage),
|
||||
PortableFunctionCallingOptions.builder()
|
||||
.withModel(AnthropicApi.ChatModel.CLAUDE_3_5_SONNET.getName())
|
||||
.build()));
|
||||
|
||||
assertThat(response.getResult().getOutput().getContent()).containsAnyOf("Spring AI", "portable API");
|
||||
}
|
||||
|
||||
@Test
|
||||
void functionCallTest() {
|
||||
|
||||
@@ -333,7 +352,8 @@ class AnthropicChatModelIT {
|
||||
// @formatter:on
|
||||
|
||||
logger.info(response.toString());
|
||||
validateChatResponseMetadata(response, model);
|
||||
// Note, brittle test.
|
||||
validateChatResponseMetadata(response, "claude-3-5-sonnet-20241022");
|
||||
}
|
||||
|
||||
record ActorsFilmsRecord(String actor, List<String> movies) {
|
||||
|
||||
Binary file not shown.
@@ -258,8 +258,6 @@ class VertexAiGeminiChatModelIT {
|
||||
|
||||
var response = this.chatModel.call(new Prompt(List.of(userMessage)));
|
||||
|
||||
System.out.println(response.getResult().getOutput().getContent());
|
||||
|
||||
assertThat(response.getResult().getOutput().getContent()).containsAnyOf("Spring AI", "portable API");
|
||||
}
|
||||
|
||||
|
||||
@@ -146,10 +146,12 @@ Read more about xref:api/chat/functions/anthropic-chat-functions.adoc[Anthropic
|
||||
|
||||
== Multimodal
|
||||
|
||||
Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. This paradigm represents a significant advancement in AI models.
|
||||
Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, pdf, images, data formats.
|
||||
|
||||
=== Images
|
||||
Currently, Anthropic Claude 3 supports the `base64` source type for `images`, and the `image/jpeg`, `image/png`, `image/gif`, and `image/webp` media types.
|
||||
Check the link:https://docs.anthropic.com/claude/docs/vision[Vision guide] for more information.
|
||||
Anthropic Claude 3.5 Sonnet also supports the `pdf` source type for `application/pdf` files.
|
||||
|
||||
Spring AI's `Message` interface supports multimodal AI models by introducing the Media type.
|
||||
This type contains data and information about media attachments in messages, using Spring's `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data.
|
||||
@@ -176,14 +178,23 @@ along with the text message "Explain what do you see on this picture?", and gene
|
||||
|
||||
----
|
||||
The image shows a close-up view of a wire fruit basket containing several pieces of fruit.
|
||||
The basket appears to be made of thin metal wires formed into a round shape with an elevated handle.
|
||||
...
|
||||
----
|
||||
|
||||
Inside the basket, there are a few yellow bananas and a couple of red apples or possibly tomatoes.
|
||||
The vibrant colors of the fruit contrast nicely against the metallic tones of the wire basket.
|
||||
=== PDF
|
||||
|
||||
The shallow depth of field in the photograph puts the focus squarely on the fruit in the foreground, while the basket handle extending upwards is slightly blurred, creating a pleasing bokeh effect in the background.
|
||||
Starting with Sonnet 3.5 https://docs.anthropic.com/en/docs/build-with-claude/pdf-support[PDF support (beta)] is provided.
|
||||
Use the `application/pdf` media type to attach a PDF file to the message:
|
||||
|
||||
The composition and lighting give the image a clean, minimalist aesthetic that highlights the natural beauty and freshness of the fruit displayed in this elegant wire basket.
|
||||
[source,java]
|
||||
----
|
||||
var pdfData = new ClassPathResource("/spring-ai-reference-overview.pdf");
|
||||
|
||||
var userMessage = new UserMessage(
|
||||
"You are a very professional document summarization specialist. Please summarize the given document.",
|
||||
List.of(new Media(new MimeType("application", "pdf"), pdfData)));
|
||||
|
||||
var response = this.chatModel.call(new Prompt(List.of(userMessage)));
|
||||
----
|
||||
|
||||
== Sample Controller
|
||||
@@ -195,7 +206,7 @@ Add a `application.properties` file, under the `src/main/resources` directory, t
|
||||
[source,application.properties]
|
||||
----
|
||||
spring.ai.anthropic.api-key=YOUR_API_KEY
|
||||
spring.ai.anthropic.chat.options.model=claude-3-5-sonnet-20241022
|
||||
spring.ai.anthropic.chat.options.model=claude-3-5-sonnet-latest
|
||||
spring.ai.anthropic.chat.options.temperature=0.7
|
||||
spring.ai.anthropic.chat.options.max-tokens=450
|
||||
----
|
||||
|
||||
@@ -19,7 +19,7 @@ This table compares various Chat Models supported by Spring AI, detailing their
|
||||
|====
|
||||
| Provider | Multimodality ^| Tools/Functions ^| Streaming ^| Retry ^| Observability ^| Built-in JSON ^| Local ^| OpenAI API Compatible
|
||||
|
||||
| xref::api/chat/anthropic-chat.adoc[Anthropic Claude] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12]
|
||||
| xref::api/chat/anthropic-chat.adoc[Anthropic Claude] | text, pdf, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12]
|
||||
| xref::api/chat/azure-openai-chat.adoc[Azure OpenAI] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
|
||||
| xref::api/chat/vertexai-gemini-chat.adoc[Google VertexAI Gemini] | text, pdf, image, audio, video ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
|
||||
| xref::api/chat/groq-chat.adoc[Groq (OpenAI-proxy)] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
|
||||
|
||||
@@ -125,8 +125,8 @@ Read more about xref:api/chat/functions/vertexai-gemini-chat-functions.adoc[Vert
|
||||
== Multimodal
|
||||
|
||||
Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including `text`, `pdf`, `images`, `audio`, and other data formats.
|
||||
This paradigm represents a significant advancement in AI models.
|
||||
|
||||
=== Image, Audio, Video
|
||||
Google's Gemini AI models support this capability by comprehending and integrating text, code, audio, images, and video.
|
||||
For more details, refer to the blog post https://blog.google/technology/ai/google-gemini-ai/#introducing-gemini[Introducing Gemini].
|
||||
|
||||
@@ -146,6 +146,23 @@ var userMessage = new UserMessage("Explain what do you see on this picture?",
|
||||
ChatResponse response = chatModel.call(new Prompt(List.of(this.userMessage)));
|
||||
----
|
||||
|
||||
=== PDF
|
||||
|
||||
Latest Vertex Gemini provides support for PDF input types..
|
||||
Use the `application/pdf` media type to attach a PDF file to the message:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
var pdfData = new ClassPathResource("/spring-ai-reference-overview.pdf");
|
||||
|
||||
var userMessage = new UserMessage(
|
||||
"You are a very professional document summarization specialist. Please summarize the given document.",
|
||||
List.of(new Media(new MimeType("application", "pdf"), pdfData)));
|
||||
|
||||
var response = this.chatModel.call(new Prompt(List.of(userMessage)));
|
||||
----
|
||||
|
||||
|
||||
== Sample Controller
|
||||
|
||||
https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-vertex-ai-gemini-spring-boot-starter` to your pom (or gradle) dependencies.
|
||||
|
||||
Reference in New Issue
Block a user