From 30eb3ce3f2690482481e317d5ab396415c035eec Mon Sep 17 00:00:00 2001 From: Thomas Vitale Date: Fri, 9 May 2025 15:30:30 +0100 Subject: [PATCH] Micrometer instrumentation for tool calling Introduce instrumentation for tool calling using the Micrometer Observation API. By default, metadata about tool calling are exported as metrics and traces. Optionally, the actual tool call input and result can be exported as well by enabling the dedicated feature flag. Signed-off-by: Thomas Vitale --- .../ToolCallingAutoConfiguration.java | 35 +++- .../autoconfigure/ToolCallingProperties.java | 51 ++++++ .../ToolCallingAutoConfigurationTests.java | 20 +++ .../conventions/AiObservationAttributes.java | 4 + .../observation/conventions/SpringAiKind.java | 5 + .../ROOT/pages/observability/index.adoc | 41 +++++ .../ChatModelObservationDocumentation.java | 10 ++ ...DefaultChatModelObservationConvention.java | 25 ++- .../model/tool/DefaultToolCallingManager.java | 45 ++++- .../ai/tool/definition/ToolDefinition.java | 7 + ...faultToolCallingObservationConvention.java | 106 ++++++++++++ .../ToolCallingContentObservationFilter.java | 52 ++++++ .../ToolCallingObservationContext.java | 129 ++++++++++++++ .../ToolCallingObservationConvention.java | 35 ++++ .../ToolCallingObservationDocumentation.java | 148 ++++++++++++++++ .../ai/tool/observation/package-info.java | 25 +++ ...ltChatModelObservationConventionTests.java | 56 ++++++ .../tool/DefaultToolCallingManagerIT.java | 159 ++++++++++++++++++ ...ToolCallingObservationConventionTests.java | 100 +++++++++++ ...lCallingContentObservationFilterTests.java | 76 +++++++++ .../ToolCallingObservationContextTests.java | 75 +++++++++ 21 files changed, 1186 insertions(+), 18 deletions(-) create mode 100644 auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingProperties.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConvention.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilter.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationContext.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationConvention.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationDocumentation.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/tool/observation/package-info.java create mode 100644 spring-ai-model/src/test/java/org/springframework/ai/model/tool/DefaultToolCallingManagerIT.java create mode 100644 spring-ai-model/src/test/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConventionTests.java create mode 100644 spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilterTests.java create mode 100644 spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingObservationContextTests.java diff --git a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java index 0090a6610..4186ce0f8 100644 --- a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java +++ b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java @@ -16,17 +16,17 @@ package org.springframework.ai.model.tool.autoconfigure; -import java.util.ArrayList; -import java.util.List; - import io.micrometer.observation.ObservationRegistry; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.model.tool.ToolCallingManager; import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.ToolCallbackProvider; import org.springframework.ai.tool.execution.DefaultToolExecutionExceptionProcessor; import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor; +import org.springframework.ai.tool.observation.ToolCallingContentObservationFilter; +import org.springframework.ai.tool.observation.ToolCallingObservationConvention; import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver; import org.springframework.ai.tool.resolution.SpringBeanToolCallbackResolver; import org.springframework.ai.tool.resolution.StaticToolCallbackResolver; @@ -35,9 +35,14 @@ import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.support.GenericApplicationContext; +import java.util.ArrayList; +import java.util.List; + /** * Auto-configuration for common tool calling features of {@link ChatModel}. * @@ -47,8 +52,11 @@ import org.springframework.context.support.GenericApplicationContext; */ @AutoConfiguration @ConditionalOnClass(ChatModel.class) +@EnableConfigurationProperties(ToolCallingProperties.class) public class ToolCallingAutoConfiguration { + private static final Logger logger = LoggerFactory.getLogger(ToolCallingAutoConfiguration.class); + @Bean @ConditionalOnMissingBean ToolCallbackResolver toolCallbackResolver(GenericApplicationContext applicationContext, @@ -76,12 +84,27 @@ public class ToolCallingAutoConfiguration { @ConditionalOnMissingBean ToolCallingManager toolCallingManager(ToolCallbackResolver toolCallbackResolver, ToolExecutionExceptionProcessor toolExecutionExceptionProcessor, - ObjectProvider observationRegistry) { - return ToolCallingManager.builder() + ObjectProvider observationRegistry, + ObjectProvider observationConvention) { + var toolCallingManager = ToolCallingManager.builder() .observationRegistry(observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP)) .toolCallbackResolver(toolCallbackResolver) .toolExecutionExceptionProcessor(toolExecutionExceptionProcessor) .build(); + + observationConvention.ifAvailable(toolCallingManager::setObservationConvention); + + return toolCallingManager; + } + + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = ToolCallingProperties.CONFIG_PREFIX + ".observations", name = "include-content", + havingValue = "true") + ToolCallingContentObservationFilter toolCallingContentObservationFilter() { + logger.warn( + "You have enabled the inclusion of the tool call arguments and result in the observations, with the risk of exposing sensitive or private information. Please, be careful!"); + return new ToolCallingContentObservationFilter(); } } diff --git a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingProperties.java b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingProperties.java new file mode 100644 index 000000000..0037a45b6 --- /dev/null +++ b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingProperties.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.model.tool.autoconfigure; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Configuration properties for tool calling. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +@ConfigurationProperties(ToolCallingProperties.CONFIG_PREFIX) +public class ToolCallingProperties { + + public static final String CONFIG_PREFIX = "spring.ai.tools"; + + private final Observations observations = new Observations(); + + public static class Observations { + + /** + * Whether to include the tool call content in the observations. + */ + private boolean includeContent = false; + + public boolean isIncludeContent() { + return includeContent; + } + + public void setIncludeContent(boolean includeContent) { + this.includeContent = includeContent; + } + + } + +} diff --git a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java index 637f9c755..3be837405 100644 --- a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java +++ b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java @@ -31,6 +31,7 @@ import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor; import org.springframework.ai.tool.function.FunctionToolCallback; import org.springframework.ai.tool.method.MethodToolCallback; import org.springframework.ai.tool.method.MethodToolCallbackProvider; +import org.springframework.ai.tool.observation.ToolCallingContentObservationFilter; import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver; import org.springframework.ai.tool.resolution.ToolCallbackResolver; import org.springframework.ai.tool.support.ToolDefinitions; @@ -111,6 +112,25 @@ class ToolCallingAutoConfigurationTests { }); } + @Test + void observationFilterDefault() { + new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(ToolCallingAutoConfiguration.class)) + .withUserConfiguration(Config.class) + .run(context -> { + assertThat(context).doesNotHaveBean(ToolCallingContentObservationFilter.class); + }); + } + + @Test + void observationFilterEnabled() { + new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(ToolCallingAutoConfiguration.class)) + .withPropertyValues("spring.ai.tools.observations.include-content=true") + .withUserConfiguration(Config.class) + .run(context -> { + assertThat(context).hasSingleBean(ToolCallingContentObservationFilter.class); + }); + } + static class WeatherService { @Tool(description = "Get the weather in location. Return temperature in 36°F or 36°C format.") diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java index 91b2eaf9c..8fbdb8c91 100644 --- a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java @@ -67,6 +67,10 @@ public enum AiObservationAttributes { * The temperature setting for the model request. */ REQUEST_TEMPERATURE("gen_ai.request.temperature"), + /** + * List of tool definitions provided to the model in the request. + */ + REQUEST_TOOL_NAMES("spring.ai.model.request.tool.names"), /** * The top_k sampling setting for the model request. */ diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java index 4f0a4b28e..0823609cc 100644 --- a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java @@ -37,6 +37,11 @@ public enum SpringAiKind { */ CHAT_CLIENT("chat_client"), + /** + * Spring AI kind for tool calling. + */ + TOOL_CALL("tool_call"), + /** * Spring AI kind for vector store. */ diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc index f166ac5a5..e0341e468 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc @@ -155,6 +155,7 @@ IMPORTANT: The `gen_ai.client.token.usage` metrics measures number of input and |`gen_ai.usage.total_tokens` | The total number of tokens used in the model exchange. |`gen_ai.prompt` | The full prompt sent to the model. Optional. |`gen_ai.completion` | The full response received from the model. Optional. +|`spring.ai.model.request.tool.names` | List of tool definitions provided to the model in the request. |=== NOTE: For measuring user tokens, the previous table lists the values present in an observation trace. @@ -179,6 +180,46 @@ Spring AI supports logging chat prompt and completion data, useful for troublesh WARNING: If you enable logging of the chat prompt and completion data, there's a risk of exposing sensitive or private information. Please, be careful! +== Tool Calling + +The `spring.ai.tool` observations are recorded when performing tool calling in the context of a chat model interaction. They measure the time spent on toll call completion and propagate the related tracing information. + +.Low Cardinality Keys +[cols="a,a", stripes=even] +|=== +|Name | Description + +|`gen_ai.operation.name` | The name of the operation being performed. It's always `framework`. +|`gen_ai.system` | The provider responsible for the operation. It's always `spring_ai`. +|`spring.ai.kind` | The kind of operation performed by Spring AI. It's always `tool_call`. +|`spring.ai.tool.definition.name` | The name of the tool. +|=== + +.High Cardinality Keys +[cols="a,a", stripes=even] +|=== +|Name | Description +|`spring.ai.tool.definition.description` | Description of the tool. +|`spring.ai.tool.definition.schema` | Schema of the parameters used to call the tool. +|`spring.ai.tool.call.arguments` | The input arguments to the tool call. (Only when enabled) +|`spring.ai.tool.call.result` | Schema of the parameters used to call the tool. (Only when enabled) +|=== + +=== Tool Call Arguments and Result Data + +The input arguments and result from the tool call are not exported by default, as they can be potentially sensitive. + +Spring AI supports exporting tool call arguments and result data as span attributes. + +[cols="6,3,1", stripes=even] +|==== +| Property | Description | Default + +| `spring.ai.tools.observations.include-content` | Include the tool call content in observations. `true` or `false` | `false` +|==== + +WARNING: If you enable the inclusion of the tool call arguments and result in the observations, there's a risk of exposing sensitive or private information. Please, be careful! + == EmbeddingModel NOTE: Observability features are currently supported only for `EmbeddingModel` implementations from the following diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java index 9f175608b..e25cddf69 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java @@ -150,6 +150,16 @@ public enum ChatModelObservationDocumentation implements ObservationDocumentatio } }, + /** + * List of tool definitions provided to the model in the request. + */ + REQUEST_TOOL_NAMES { + @Override + public String asString() { + return AiObservationAttributes.REQUEST_TOOL_NAMES.value(); + } + }, + /** * The top_k sampling setting for the model request. */ diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java index 8ebb8c2b0..2f215e77d 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java @@ -16,13 +16,15 @@ package org.springframework.ai.chat.observation; -import java.util.Objects; +import java.util.HashSet; +import java.util.Set; import java.util.StringJoiner; import io.micrometer.common.KeyValue; import io.micrometer.common.KeyValues; import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.model.tool.ToolCallingChatOptions; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; @@ -100,6 +102,7 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati keyValues = requestPresencePenalty(keyValues, context); keyValues = requestStopSequences(keyValues, context); keyValues = requestTemperature(keyValues, context); + keyValues = requestTools(keyValues, context); keyValues = requestTopK(keyValues, context); keyValues = requestTopP(keyValues, context); // Response @@ -148,8 +151,6 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati if (!CollectionUtils.isEmpty(options.getStopSequences())) { StringJoiner stopSequencesJoiner = new StringJoiner(", ", "[", "]"); options.getStopSequences().forEach(value -> stopSequencesJoiner.add("\"" + value + "\"")); - KeyValue.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES, - options.getStopSequences(), Objects::nonNull); return keyValues.and( ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES.asString(), stopSequencesJoiner.toString()); @@ -167,6 +168,24 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati return keyValues; } + protected KeyValues requestTools(KeyValues keyValues, ChatModelObservationContext context) { + if (!(context.getRequest().getOptions() instanceof ToolCallingChatOptions options)) { + return keyValues; + } + + Set toolNames = new HashSet<>(options.getToolNames()); + toolNames.addAll(options.getToolCallbacks().stream().map(tc -> tc.getToolDefinition().name()).toList()); + + if (!CollectionUtils.isEmpty(toolNames)) { + StringJoiner toolNamesJoiner = new StringJoiner(", ", "[", "]"); + toolNames.forEach(value -> toolNamesJoiner.add("\"" + value + "\"")); + return keyValues.and( + ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString(), + toolNamesJoiner.toString()); + } + return keyValues; + } + protected KeyValues requestTopK(KeyValues keyValues, ChatModelObservationContext context) { ChatOptions options = context.getRequest().getOptions(); if (options.getTopK() != null) { diff --git a/spring-ai-model/src/main/java/org/springframework/ai/model/tool/DefaultToolCallingManager.java b/spring-ai-model/src/main/java/org/springframework/ai/model/tool/DefaultToolCallingManager.java index e80db2974..887ba56bb 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/model/tool/DefaultToolCallingManager.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/model/tool/DefaultToolCallingManager.java @@ -38,6 +38,10 @@ import org.springframework.ai.tool.definition.ToolDefinition; import org.springframework.ai.tool.execution.DefaultToolExecutionExceptionProcessor; import org.springframework.ai.tool.execution.ToolExecutionException; import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor; +import org.springframework.ai.tool.observation.DefaultToolCallingObservationConvention; +import org.springframework.ai.tool.observation.ToolCallingObservationContext; +import org.springframework.ai.tool.observation.ToolCallingObservationConvention; +import org.springframework.ai.tool.observation.ToolCallingObservationDocumentation; import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver; import org.springframework.ai.tool.resolution.ToolCallbackResolver; import org.springframework.util.Assert; @@ -58,6 +62,9 @@ public final class DefaultToolCallingManager implements ToolCallingManager { private static final ObservationRegistry DEFAULT_OBSERVATION_REGISTRY = ObservationRegistry.NOOP; + private static final ToolCallingObservationConvention DEFAULT_OBSERVATION_CONVENTION + = new DefaultToolCallingObservationConvention(); + private static final ToolCallbackResolver DEFAULT_TOOL_CALLBACK_RESOLVER = new DelegatingToolCallbackResolver(List.of()); @@ -72,6 +79,8 @@ public final class DefaultToolCallingManager implements ToolCallingManager { private final ToolExecutionExceptionProcessor toolExecutionExceptionProcessor; + private ToolCallingObservationConvention observationConvention = DEFAULT_OBSERVATION_CONVENTION; + public DefaultToolCallingManager(ObservationRegistry observationRegistry, ToolCallbackResolver toolCallbackResolver, ToolExecutionExceptionProcessor toolExecutionExceptionProcessor) { Assert.notNull(observationRegistry, "observationRegistry cannot be null"); @@ -104,7 +113,7 @@ public final class DefaultToolCallingManager implements ToolCallingManager { toolCallbacks.add(toolCallback); } - return toolCallbacks.stream().map(toolCallback -> toolCallback.getToolDefinition()).toList(); + return toolCallbacks.stream().map(ToolCallback::getToolDefinition).toList(); } @Override @@ -200,15 +209,29 @@ public final class DefaultToolCallingManager implements ToolCallingManager { returnDirect = returnDirect && toolCallback.getToolMetadata().returnDirect(); } - String toolResult; - try { - toolResult = toolCallback.call(toolInputArguments, toolContext); - } - catch (ToolExecutionException ex) { - toolResult = this.toolExecutionExceptionProcessor.process(ex); - } + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(toolCallback.getToolDefinition()) + .toolMetadata(toolCallback.getToolMetadata()) + .toolCallArguments(toolInputArguments) + .build(); - toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolName, toolResult)); + String toolCallResult = ToolCallingObservationDocumentation.TOOL_CALL + .observation(this.observationConvention, DEFAULT_OBSERVATION_CONVENTION, () -> observationContext, + this.observationRegistry) + .observe(() -> { + String toolResult; + try { + toolResult = toolCallback.call(toolInputArguments, toolContext); + } + catch (ToolExecutionException ex) { + toolResult = this.toolExecutionExceptionProcessor.process(ex); + } + observationContext.setToolCallResult(toolResult); + return toolResult; + }); + + toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolName, + toolCallResult != null ? toolCallResult : "")); } return new InternalToolExecutionResult(new ToolResponseMessage(toolResponses, Map.of()), returnDirect); @@ -222,6 +245,10 @@ public final class DefaultToolCallingManager implements ToolCallingManager { return messages; } + public void setObservationConvention(ToolCallingObservationConvention observationConvention) { + this.observationConvention = observationConvention; + } + public static Builder builder() { return new Builder(); } diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/definition/ToolDefinition.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/definition/ToolDefinition.java index 089c7b305..517a00617 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/tool/definition/ToolDefinition.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/definition/ToolDefinition.java @@ -39,4 +39,11 @@ public interface ToolDefinition { */ String inputSchema(); + /** + * Create a default {@link ToolDefinition} builder. + */ + static DefaultToolDefinition.Builder builder() { + return DefaultToolDefinition.builder(); + } + } diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConvention.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConvention.java new file mode 100644 index 000000000..0842ec5de --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConvention.java @@ -0,0 +1,106 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.common.KeyValue; +import io.micrometer.common.KeyValues; +import org.springframework.ai.observation.conventions.SpringAiKind; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Default conventions to populate observations for tool calling operations. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public class DefaultToolCallingObservationConvention implements ToolCallingObservationConvention { + + public static final String DEFAULT_NAME = "spring.ai.tool"; + + private final String name; + + public DefaultToolCallingObservationConvention() { + this(DEFAULT_NAME); + } + + public DefaultToolCallingObservationConvention(String name) { + this.name = name; + } + + @Override + public String getName() { + return this.name; + } + + @Override + @Nullable + public String getContextualName(ToolCallingObservationContext context) { + Assert.notNull(context, "context cannot be null"); + String toolName = context.getToolDefinition().name(); + return "%s %s".formatted(SpringAiKind.TOOL_CALL.value(), toolName); + } + + @Override + public KeyValues getLowCardinalityKeyValues(ToolCallingObservationContext context) { + return KeyValues.of(aiOperationType(context), aiProvider(context), springAiKind(context), + toolDefinitionName(context)); + } + + protected KeyValue aiOperationType(ToolCallingObservationContext context) { + return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE, + context.getOperationMetadata().operationType()); + } + + protected KeyValue aiProvider(ToolCallingObservationContext context) { + return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER, + context.getOperationMetadata().provider()); + } + + protected KeyValue springAiKind(ToolCallingObservationContext context) { + return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND, + SpringAiKind.TOOL_CALL.value()); + } + + protected KeyValue toolDefinitionName(ToolCallingObservationContext context) { + String toolName = context.getToolDefinition().name(); + return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME, toolName); + } + + @Override + public KeyValues getHighCardinalityKeyValues(ToolCallingObservationContext context) { + var keyValues = KeyValues.empty(); + keyValues = toolDefinitionDescription(keyValues, context); + keyValues = toolDefinitionSchema(keyValues, context); + return keyValues; + } + + protected KeyValues toolDefinitionDescription(KeyValues keyValues, ToolCallingObservationContext context) { + String toolDescription = context.getToolDefinition().description(); + return keyValues.and( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION.asString(), + toolDescription); + } + + protected KeyValues toolDefinitionSchema(KeyValues keyValues, ToolCallingObservationContext context) { + String toolSchema = context.getToolDefinition().inputSchema(); + return keyValues.and( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(), + toolSchema); + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilter.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilter.java new file mode 100644 index 000000000..9f1e5547e --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilter.java @@ -0,0 +1,52 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationFilter; + +/** + * An {@link ObservationFilter} to include the tool call content (input/output) in the + * observation. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public class ToolCallingContentObservationFilter implements ObservationFilter { + + @Override + public Observation.Context map(Observation.Context context) { + if (!(context instanceof ToolCallingObservationContext toolCallingObservationContext)) { + return context; + } + + String toolCallArguments = toolCallingObservationContext.getToolCallArguments(); + toolCallingObservationContext + .addHighCardinalityKeyValue(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS + .withValue(toolCallArguments)); + + String toolCallResult = toolCallingObservationContext.getToolCallResult(); + if (toolCallResult != null) { + toolCallingObservationContext + .addHighCardinalityKeyValue(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT + .withValue(toolCallResult)); + } + + return toolCallingObservationContext; + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationContext.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationContext.java new file mode 100644 index 000000000..3ce781824 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationContext.java @@ -0,0 +1,129 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.observation.Observation; +import org.springframework.ai.observation.AiOperationMetadata; +import org.springframework.ai.observation.conventions.AiOperationType; +import org.springframework.ai.observation.conventions.AiProvider; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Context used to store data for tool calling observations. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public class ToolCallingObservationContext extends Observation.Context { + + private final AiOperationMetadata operationMetadata = new AiOperationMetadata(AiOperationType.FRAMEWORK.value(), + AiProvider.SPRING_AI.value()); + + private final ToolDefinition toolDefinition; + + private final ToolMetadata toolMetadata; + + private final String toolCallArguments; + + @Nullable + private String toolCallResult; + + private ToolCallingObservationContext(ToolDefinition toolDefinition, ToolMetadata toolMetadata, + String toolCallArguments, @Nullable String toolCallResult) { + Assert.notNull(toolDefinition, "toolDefinition cannot be null"); + Assert.notNull(toolMetadata, "toolMetadata cannot be null"); + Assert.hasText(toolCallArguments, "toolCallArguments cannot be null or empty"); + + this.toolDefinition = toolDefinition; + this.toolMetadata = toolMetadata; + this.toolCallArguments = toolCallArguments; + this.toolCallResult = toolCallResult; + } + + public AiOperationMetadata getOperationMetadata() { + return operationMetadata; + } + + public ToolDefinition getToolDefinition() { + return toolDefinition; + } + + public ToolMetadata getToolMetadata() { + return toolMetadata; + } + + public String getToolCallArguments() { + return toolCallArguments; + } + + @Nullable + public String getToolCallResult() { + return toolCallResult; + } + + public void setToolCallResult(@Nullable String toolCallResult) { + this.toolCallResult = toolCallResult; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private ToolDefinition toolDefinition; + + private ToolMetadata toolMetadata = ToolMetadata.builder().build(); + + private String toolCallArguments; + + @Nullable + private String toolCallResult; + + private Builder() { + } + + public Builder toolDefinition(ToolDefinition toolDefinition) { + this.toolDefinition = toolDefinition; + return this; + } + + public Builder toolMetadata(ToolMetadata toolMetadata) { + this.toolMetadata = toolMetadata; + return this; + } + + public Builder toolCallArguments(String toolCallArguments) { + this.toolCallArguments = toolCallArguments; + return this; + } + + public Builder toolCallResult(@Nullable String toolCallResult) { + this.toolCallResult = toolCallResult; + return this; + } + + public ToolCallingObservationContext build() { + return new ToolCallingObservationContext(toolDefinition, toolMetadata, toolCallArguments, toolCallResult); + } + + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationConvention.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationConvention.java new file mode 100644 index 000000000..28481a9bb --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationConvention.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationConvention; + +/** + * Interface for an {@link ObservationConvention} for tool calling observations. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public interface ToolCallingObservationConvention extends ObservationConvention { + + @Override + default boolean supportsContext(Observation.Context context) { + return context instanceof ToolCallingObservationContext; + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationDocumentation.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationDocumentation.java new file mode 100644 index 000000000..dd6447c92 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationDocumentation.java @@ -0,0 +1,148 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.common.docs.KeyName; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationConvention; +import io.micrometer.observation.docs.ObservationDocumentation; +import org.springframework.ai.observation.conventions.AiObservationAttributes; + +/** + * Tool calling observation documentation. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public enum ToolCallingObservationDocumentation implements ObservationDocumentation { + + /** + * Tool calling observations. + */ + TOOL_CALL { + @Override + public Class> getDefaultConvention() { + return DefaultToolCallingObservationConvention.class; + } + + @Override + public KeyName[] getLowCardinalityKeyNames() { + return LowCardinalityKeyNames.values(); + } + + @Override + public KeyName[] getHighCardinalityKeyNames() { + return HighCardinalityKeyNames.values(); + } + + }; + + /** + * Low cardinality key names. + */ + public enum LowCardinalityKeyNames implements KeyName { + + /** + * The name of the operation being performed. + */ + AI_OPERATION_TYPE { + @Override + public String asString() { + return AiObservationAttributes.AI_OPERATION_TYPE.value(); + } + }, + + /** + * The provider responsible for the operation. + */ + AI_PROVIDER { + @Override + public String asString() { + return AiObservationAttributes.AI_PROVIDER.value(); + } + }, + + /** + * Spring AI kind. + */ + SPRING_AI_KIND { + @Override + public String asString() { + return "spring.ai.kind"; + } + }, + + /** + * The name of the tool. + */ + TOOL_DEFINITION_NAME { + @Override + public String asString() { + return "spring.ai.tool.definition.name"; + } + }, + + } + + /** + * High cardinality key names. + */ + public enum HighCardinalityKeyNames implements KeyName { + + /** + * Description of the tool. + */ + TOOL_DEFINITION_DESCRIPTION { + @Override + public String asString() { + return "spring.ai.tool.definition.description"; + } + }, + + /** + * Schema of the parameters used to call the tool. + */ + TOOL_DEFINITION_SCHEMA { + @Override + public String asString() { + return "spring.ai.tool.definition.schema"; + } + }, + + /** + * The input arguments to the tool call. + */ + TOOL_CALL_ARGUMENTS { + @Override + public String asString() { + return "spring.ai.tool.call.arguments"; + } + }, + + /** + * The result of the tool call. + */ + TOOL_CALL_RESULT { + @Override + public String asString() { + return "spring.ai.tool.call.result"; + } + } + + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/package-info.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/package-info.java new file mode 100644 index 000000000..4708201ec --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/package-info.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides the API for chat client advisors observations. + */ +@NonNullApi +@NonNullFields +package org.springframework.ai.tool.observation; + +import org.springframework.lang.NonNullApi; +import org.springframework.lang.NonNullFields; diff --git a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java index 5629a1de4..da12a2490 100644 --- a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java +++ b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java @@ -32,6 +32,11 @@ import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.model.tool.ToolCallingChatOptions; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.ai.chat.observation.ChatModelObservationDocumentation.HighCardinalityKeyNames; @@ -145,6 +150,7 @@ class DefaultChatModelObservationConventionTests { HighCardinalityKeyNames.REQUEST_PRESENCE_PENALTY.asString(), HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES.asString(), HighCardinalityKeyNames.REQUEST_TEMPERATURE.asString(), + HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString(), HighCardinalityKeyNames.REQUEST_TOP_K.asString(), HighCardinalityKeyNames.REQUEST_TOP_P.asString(), HighCardinalityKeyNames.RESPONSE_FINISH_REASONS.asString(), HighCardinalityKeyNames.RESPONSE_ID.asString(), @@ -171,6 +177,23 @@ class DefaultChatModelObservationConventionTests { HighCardinalityKeyNames.RESPONSE_ID.asString()); } + @Test + void shouldHaveKeyValuesWhenTools() { + ChatModelObservationContext observationContext = ChatModelObservationContext.builder() + .prompt(generatePrompt(ToolCallingChatOptions.builder() + .model("mistral") + .toolNames("toolA", "toolB") + .toolCallbacks(new TestToolCallback("tool1", true), new TestToolCallback("tool2", false), + new TestToolCallback("toolB")) + .build())) + .provider("superprovider") + .build(); + assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)).anySatisfy(keyValue -> { + assertThat(keyValue.getKey()).isEqualTo(HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString()); + assertThat(keyValue.getValue()).contains("toolA", "toolB", "tool1", "tool2"); + }); + } + private Prompt generatePrompt(ChatOptions chatOptions) { return new Prompt("Who let the dogs out?", chatOptions); } @@ -198,4 +221,37 @@ class DefaultChatModelObservationConventionTests { } + static class TestToolCallback implements ToolCallback { + + private final ToolDefinition toolDefinition; + + private final ToolMetadata toolMetadata; + + TestToolCallback(String name) { + this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build(); + this.toolMetadata = ToolMetadata.builder().build(); + } + + TestToolCallback(String name, boolean returnDirect) { + this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build(); + this.toolMetadata = ToolMetadata.builder().returnDirect(returnDirect).build(); + } + + @Override + public ToolDefinition getToolDefinition() { + return this.toolDefinition; + } + + @Override + public ToolMetadata getToolMetadata() { + return this.toolMetadata; + } + + @Override + public String call(String toolInput) { + return "Mission accomplished!"; + } + + } + } diff --git a/spring-ai-model/src/test/java/org/springframework/ai/model/tool/DefaultToolCallingManagerIT.java b/spring-ai-model/src/test/java/org/springframework/ai/model/tool/DefaultToolCallingManagerIT.java new file mode 100644 index 000000000..e35d5a9a8 --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/model/tool/DefaultToolCallingManagerIT.java @@ -0,0 +1,159 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.model.tool; + +import io.micrometer.observation.tck.TestObservationRegistry; +import io.micrometer.observation.tck.TestObservationRegistryAssert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.metadata.ChatResponseMetadata; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.observation.conventions.AiOperationType; +import org.springframework.ai.observation.conventions.AiProvider; +import org.springframework.ai.observation.conventions.SpringAiKind; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; +import org.springframework.ai.tool.observation.DefaultToolCallingObservationConvention; +import org.springframework.ai.tool.observation.ToolCallingObservationDocumentation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests for {@link DefaultToolCallingManager}. + * + * @author Thomas Vitale + */ +@SpringBootTest(classes = DefaultToolCallingManagerIT.Config.class) +class DefaultToolCallingManagerIT { + + @Autowired + TestObservationRegistry observationRegistry; + + @Autowired + ToolCallingManager toolCallingManager; + + @BeforeEach + void beforeEach() { + this.observationRegistry.clear(); + } + + @Test + void observationForToolCall() { + ToolCallback toolCallback = new TestToolCallback("toolA"); + Prompt prompt = Prompt.builder() + .content("Why does a raven look like a desk?") + .chatOptions(ToolCallingChatOptions.builder().toolCallbacks(toolCallback).build()) + .build(); + + ChatResponse chatResponse = ChatResponse.builder() + .generations(List.of(new Generation(new AssistantMessage("Answer", Map.of(), + List.of(new AssistantMessage.ToolCall("toolA", "function", "toolA", "{}")))))) + .build(); + + ToolExecutionResult toolExecutionResult = toolCallingManager.executeToolCalls(prompt, chatResponse); + + assertThat(toolExecutionResult).isNotNull(); + + ChatResponseMetadata responseMetadata = chatResponse.getMetadata(); + assertThat(responseMetadata).isNotNull(); + + TestObservationRegistryAssert.assertThat(this.observationRegistry) + .doesNotHaveAnyRemainingCurrentObservation() + .hasObservationWithNameEqualTo(DefaultToolCallingObservationConvention.DEFAULT_NAME) + .that() + .hasContextualNameEqualTo(SpringAiKind.TOOL_CALL.value() + " " + toolCallback.getToolDefinition().name()) + .hasLowCardinalityKeyValue( + ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE.asString(), + AiOperationType.FRAMEWORK.value()) + .hasLowCardinalityKeyValue( + ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER.asString(), + AiProvider.SPRING_AI.value()) + .hasLowCardinalityKeyValue( + ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND.asString(), + SpringAiKind.TOOL_CALL.value()) + .hasLowCardinalityKeyValue( + ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME.asString(), + toolCallback.getToolDefinition().name()) + .hasHighCardinalityKeyValue( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION.asString(), + toolCallback.getToolDefinition().description()) + .hasHighCardinalityKeyValue( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(), + toolCallback.getToolDefinition().inputSchema()); + } + + @SpringBootConfiguration + static class Config { + + @Bean + public TestObservationRegistry observationRegistry() { + return TestObservationRegistry.create(); + } + + @Bean + public ToolCallingManager toolCallingManager(TestObservationRegistry observationRegistry) { + return DefaultToolCallingManager.builder().observationRegistry(observationRegistry).build(); + } + + } + + static class TestToolCallback implements ToolCallback { + + private final ToolDefinition toolDefinition; + + private final ToolMetadata toolMetadata; + + TestToolCallback(String name) { + this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build(); + this.toolMetadata = ToolMetadata.builder().build(); + } + + TestToolCallback(String name, boolean returnDirect) { + this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build(); + this.toolMetadata = ToolMetadata.builder().returnDirect(returnDirect).build(); + } + + @Override + public ToolDefinition getToolDefinition() { + return this.toolDefinition; + } + + @Override + public ToolMetadata getToolMetadata() { + return this.toolMetadata; + } + + @Override + public String call(String toolInput) { + return "Mission accomplished!"; + } + + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConventionTests.java b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConventionTests.java new file mode 100644 index 000000000..ebc71333b --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConventionTests.java @@ -0,0 +1,100 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.common.KeyValue; +import io.micrometer.observation.Observation; +import org.junit.jupiter.api.Test; +import org.springframework.ai.observation.conventions.AiOperationType; +import org.springframework.ai.observation.conventions.AiProvider; +import org.springframework.ai.observation.conventions.SpringAiKind; +import org.springframework.ai.tool.definition.ToolDefinition; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link DefaultToolCallingObservationConvention}. + * + * @author Thomas Vitale + */ +class DefaultToolCallingObservationConventionTests { + + private final DefaultToolCallingObservationConvention observationConvention = new DefaultToolCallingObservationConvention(); + + @Test + void shouldHaveName() { + assertThat(this.observationConvention.getName()) + .isEqualTo(DefaultToolCallingObservationConvention.DEFAULT_NAME); + } + + @Test + void contextualName() { + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .build(); + assertThat(this.observationConvention.getContextualName(observationContext)).isEqualTo("tool_call toolA"); + } + + @Test + void supportsOnlyChatModelObservationContext() { + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .build(); + assertThat(this.observationConvention.supportsContext(observationContext)).isTrue(); + assertThat(this.observationConvention.supportsContext(new Observation.Context())).isFalse(); + } + + @Test + void shouldHaveLowCardinalityKeyValues() { + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .build(); + assertThat(this.observationConvention.getLowCardinalityKeyValues(observationContext)).contains( + KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME.asString(), + "toolA"), + KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE.asString(), + AiOperationType.FRAMEWORK.value()), + KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER.asString(), + AiProvider.SPRING_AI.value()), + KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND, + SpringAiKind.TOOL_CALL.value())); + } + + @Test + void shouldHaveHighCardinalityKeyValues() { + String toolCallInput = """ + { + "lizard": "George" + } + """; + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments(toolCallInput) + .toolCallResult("Mission accomplished!") + .build(); + assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)).contains( + KeyValue.of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION + .asString(), "description"), + KeyValue.of( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(), + "{}")); + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilterTests.java b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilterTests.java new file mode 100644 index 000000000..b053946a7 --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilterTests.java @@ -0,0 +1,76 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.common.KeyValue; +import io.micrometer.observation.Observation; +import org.junit.jupiter.api.Test; +import org.springframework.ai.tool.definition.ToolDefinition; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link ToolCallingContentObservationFilter}. + * + * @author Thomas Vitale + */ +class ToolCallingContentObservationFilterTests { + + ToolCallingContentObservationFilter observationFilter = new ToolCallingContentObservationFilter(); + + @Test + void whenNotSupportedObservationContextThenReturnOriginalContext() { + var expectedContext = new Observation.Context(); + var actualContext = this.observationFilter.map(expectedContext); + + assertThat(actualContext).isEqualTo(expectedContext); + } + + @Test + void augmentContext() { + var originalContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .toolCallResult("result") + .build(); + var augmentedContext = this.observationFilter.map(originalContext); + + assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue + .of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS.asString(), "input")); + assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue + .of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT.asString(), "result")); + } + + @Test + void augmentContextWhenNullResult() { + var originalContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .toolCallResult("result") + .build(); + var augmentedContext = this.observationFilter.map(originalContext); + + assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue + .of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS.asString(), "input")); + assertThat(augmentedContext.getHighCardinalityKeyValues() + .stream() + .filter(kv -> kv.getKey() + .equals(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT.name()))) + .isEmpty(); + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingObservationContextTests.java b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingObservationContextTests.java new file mode 100644 index 000000000..cde5fc1fb --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingObservationContextTests.java @@ -0,0 +1,75 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.tool.definition.ToolDefinition; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Unit tests for {@link ToolCallingObservationContext}. + * + * @author Thomas Vitale + */ +class ToolCallingObservationContextTests { + + @Test + void whenMandatoryRequestOptionsThenReturn() { + var observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("lizard") + .build(); + assertThat(observationContext).isNotNull(); + } + + @Test + void whenToolDefinitionIsNullThenThrow() { + assertThatThrownBy(() -> ToolCallingObservationContext.builder().toolCallArguments("lizard").build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("toolDefinition cannot be null"); + } + + @Test + void whenToolMetadataIsNullThenThrow() { + assertThatThrownBy(() -> ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("lizard") + .toolMetadata(null) + .build()).isInstanceOf(IllegalArgumentException.class).hasMessageContaining("toolMetadata cannot be null"); + } + + @Test + void whenToolCallInputIsNullThenThrow() { + assertThatThrownBy(() -> ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments(null) + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("toolCallArguments cannot be null or empty"); + } + + @Test + void whenToolCallInputIsEmptyThenThrow() { + assertThatThrownBy(() -> ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("") + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("toolCallArguments cannot be null or empty"); + } + +}