diff --git a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java index 0090a6610..4186ce0f8 100644 --- a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java +++ b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java @@ -16,17 +16,17 @@ package org.springframework.ai.model.tool.autoconfigure; -import java.util.ArrayList; -import java.util.List; - import io.micrometer.observation.ObservationRegistry; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.model.tool.ToolCallingManager; import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.ToolCallbackProvider; import org.springframework.ai.tool.execution.DefaultToolExecutionExceptionProcessor; import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor; +import org.springframework.ai.tool.observation.ToolCallingContentObservationFilter; +import org.springframework.ai.tool.observation.ToolCallingObservationConvention; import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver; import org.springframework.ai.tool.resolution.SpringBeanToolCallbackResolver; import org.springframework.ai.tool.resolution.StaticToolCallbackResolver; @@ -35,9 +35,14 @@ import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.support.GenericApplicationContext; +import java.util.ArrayList; +import java.util.List; + /** * Auto-configuration for common tool calling features of {@link ChatModel}. * @@ -47,8 +52,11 @@ import org.springframework.context.support.GenericApplicationContext; */ @AutoConfiguration @ConditionalOnClass(ChatModel.class) +@EnableConfigurationProperties(ToolCallingProperties.class) public class ToolCallingAutoConfiguration { + private static final Logger logger = LoggerFactory.getLogger(ToolCallingAutoConfiguration.class); + @Bean @ConditionalOnMissingBean ToolCallbackResolver toolCallbackResolver(GenericApplicationContext applicationContext, @@ -76,12 +84,27 @@ public class ToolCallingAutoConfiguration { @ConditionalOnMissingBean ToolCallingManager toolCallingManager(ToolCallbackResolver toolCallbackResolver, ToolExecutionExceptionProcessor toolExecutionExceptionProcessor, - ObjectProvider observationRegistry) { - return ToolCallingManager.builder() + ObjectProvider observationRegistry, + ObjectProvider observationConvention) { + var toolCallingManager = ToolCallingManager.builder() .observationRegistry(observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP)) .toolCallbackResolver(toolCallbackResolver) .toolExecutionExceptionProcessor(toolExecutionExceptionProcessor) .build(); + + observationConvention.ifAvailable(toolCallingManager::setObservationConvention); + + return toolCallingManager; + } + + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = ToolCallingProperties.CONFIG_PREFIX + ".observations", name = "include-content", + havingValue = "true") + ToolCallingContentObservationFilter toolCallingContentObservationFilter() { + logger.warn( + "You have enabled the inclusion of the tool call arguments and result in the observations, with the risk of exposing sensitive or private information. Please, be careful!"); + return new ToolCallingContentObservationFilter(); } } diff --git a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingProperties.java b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingProperties.java new file mode 100644 index 000000000..0037a45b6 --- /dev/null +++ b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingProperties.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.model.tool.autoconfigure; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Configuration properties for tool calling. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +@ConfigurationProperties(ToolCallingProperties.CONFIG_PREFIX) +public class ToolCallingProperties { + + public static final String CONFIG_PREFIX = "spring.ai.tools"; + + private final Observations observations = new Observations(); + + public static class Observations { + + /** + * Whether to include the tool call content in the observations. + */ + private boolean includeContent = false; + + public boolean isIncludeContent() { + return includeContent; + } + + public void setIncludeContent(boolean includeContent) { + this.includeContent = includeContent; + } + + } + +} diff --git a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java index 637f9c755..3be837405 100644 --- a/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java +++ b/auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java @@ -31,6 +31,7 @@ import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor; import org.springframework.ai.tool.function.FunctionToolCallback; import org.springframework.ai.tool.method.MethodToolCallback; import org.springframework.ai.tool.method.MethodToolCallbackProvider; +import org.springframework.ai.tool.observation.ToolCallingContentObservationFilter; import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver; import org.springframework.ai.tool.resolution.ToolCallbackResolver; import org.springframework.ai.tool.support.ToolDefinitions; @@ -111,6 +112,25 @@ class ToolCallingAutoConfigurationTests { }); } + @Test + void observationFilterDefault() { + new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(ToolCallingAutoConfiguration.class)) + .withUserConfiguration(Config.class) + .run(context -> { + assertThat(context).doesNotHaveBean(ToolCallingContentObservationFilter.class); + }); + } + + @Test + void observationFilterEnabled() { + new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(ToolCallingAutoConfiguration.class)) + .withPropertyValues("spring.ai.tools.observations.include-content=true") + .withUserConfiguration(Config.class) + .run(context -> { + assertThat(context).hasSingleBean(ToolCallingContentObservationFilter.class); + }); + } + static class WeatherService { @Tool(description = "Get the weather in location. Return temperature in 36°F or 36°C format.") diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java index 91b2eaf9c..8fbdb8c91 100644 --- a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java @@ -67,6 +67,10 @@ public enum AiObservationAttributes { * The temperature setting for the model request. */ REQUEST_TEMPERATURE("gen_ai.request.temperature"), + /** + * List of tool definitions provided to the model in the request. + */ + REQUEST_TOOL_NAMES("spring.ai.model.request.tool.names"), /** * The top_k sampling setting for the model request. */ diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java index 4f0a4b28e..0823609cc 100644 --- a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java @@ -37,6 +37,11 @@ public enum SpringAiKind { */ CHAT_CLIENT("chat_client"), + /** + * Spring AI kind for tool calling. + */ + TOOL_CALL("tool_call"), + /** * Spring AI kind for vector store. */ diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc index f166ac5a5..e0341e468 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc @@ -155,6 +155,7 @@ IMPORTANT: The `gen_ai.client.token.usage` metrics measures number of input and |`gen_ai.usage.total_tokens` | The total number of tokens used in the model exchange. |`gen_ai.prompt` | The full prompt sent to the model. Optional. |`gen_ai.completion` | The full response received from the model. Optional. +|`spring.ai.model.request.tool.names` | List of tool definitions provided to the model in the request. |=== NOTE: For measuring user tokens, the previous table lists the values present in an observation trace. @@ -179,6 +180,46 @@ Spring AI supports logging chat prompt and completion data, useful for troublesh WARNING: If you enable logging of the chat prompt and completion data, there's a risk of exposing sensitive or private information. Please, be careful! +== Tool Calling + +The `spring.ai.tool` observations are recorded when performing tool calling in the context of a chat model interaction. They measure the time spent on toll call completion and propagate the related tracing information. + +.Low Cardinality Keys +[cols="a,a", stripes=even] +|=== +|Name | Description + +|`gen_ai.operation.name` | The name of the operation being performed. It's always `framework`. +|`gen_ai.system` | The provider responsible for the operation. It's always `spring_ai`. +|`spring.ai.kind` | The kind of operation performed by Spring AI. It's always `tool_call`. +|`spring.ai.tool.definition.name` | The name of the tool. +|=== + +.High Cardinality Keys +[cols="a,a", stripes=even] +|=== +|Name | Description +|`spring.ai.tool.definition.description` | Description of the tool. +|`spring.ai.tool.definition.schema` | Schema of the parameters used to call the tool. +|`spring.ai.tool.call.arguments` | The input arguments to the tool call. (Only when enabled) +|`spring.ai.tool.call.result` | Schema of the parameters used to call the tool. (Only when enabled) +|=== + +=== Tool Call Arguments and Result Data + +The input arguments and result from the tool call are not exported by default, as they can be potentially sensitive. + +Spring AI supports exporting tool call arguments and result data as span attributes. + +[cols="6,3,1", stripes=even] +|==== +| Property | Description | Default + +| `spring.ai.tools.observations.include-content` | Include the tool call content in observations. `true` or `false` | `false` +|==== + +WARNING: If you enable the inclusion of the tool call arguments and result in the observations, there's a risk of exposing sensitive or private information. Please, be careful! + == EmbeddingModel NOTE: Observability features are currently supported only for `EmbeddingModel` implementations from the following diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java index 9f175608b..e25cddf69 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java @@ -150,6 +150,16 @@ public enum ChatModelObservationDocumentation implements ObservationDocumentatio } }, + /** + * List of tool definitions provided to the model in the request. + */ + REQUEST_TOOL_NAMES { + @Override + public String asString() { + return AiObservationAttributes.REQUEST_TOOL_NAMES.value(); + } + }, + /** * The top_k sampling setting for the model request. */ diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java index 8ebb8c2b0..2f215e77d 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java @@ -16,13 +16,15 @@ package org.springframework.ai.chat.observation; -import java.util.Objects; +import java.util.HashSet; +import java.util.Set; import java.util.StringJoiner; import io.micrometer.common.KeyValue; import io.micrometer.common.KeyValues; import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.model.tool.ToolCallingChatOptions; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; @@ -100,6 +102,7 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati keyValues = requestPresencePenalty(keyValues, context); keyValues = requestStopSequences(keyValues, context); keyValues = requestTemperature(keyValues, context); + keyValues = requestTools(keyValues, context); keyValues = requestTopK(keyValues, context); keyValues = requestTopP(keyValues, context); // Response @@ -148,8 +151,6 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati if (!CollectionUtils.isEmpty(options.getStopSequences())) { StringJoiner stopSequencesJoiner = new StringJoiner(", ", "[", "]"); options.getStopSequences().forEach(value -> stopSequencesJoiner.add("\"" + value + "\"")); - KeyValue.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES, - options.getStopSequences(), Objects::nonNull); return keyValues.and( ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES.asString(), stopSequencesJoiner.toString()); @@ -167,6 +168,24 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati return keyValues; } + protected KeyValues requestTools(KeyValues keyValues, ChatModelObservationContext context) { + if (!(context.getRequest().getOptions() instanceof ToolCallingChatOptions options)) { + return keyValues; + } + + Set toolNames = new HashSet<>(options.getToolNames()); + toolNames.addAll(options.getToolCallbacks().stream().map(tc -> tc.getToolDefinition().name()).toList()); + + if (!CollectionUtils.isEmpty(toolNames)) { + StringJoiner toolNamesJoiner = new StringJoiner(", ", "[", "]"); + toolNames.forEach(value -> toolNamesJoiner.add("\"" + value + "\"")); + return keyValues.and( + ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString(), + toolNamesJoiner.toString()); + } + return keyValues; + } + protected KeyValues requestTopK(KeyValues keyValues, ChatModelObservationContext context) { ChatOptions options = context.getRequest().getOptions(); if (options.getTopK() != null) { diff --git a/spring-ai-model/src/main/java/org/springframework/ai/model/tool/DefaultToolCallingManager.java b/spring-ai-model/src/main/java/org/springframework/ai/model/tool/DefaultToolCallingManager.java index e80db2974..887ba56bb 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/model/tool/DefaultToolCallingManager.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/model/tool/DefaultToolCallingManager.java @@ -38,6 +38,10 @@ import org.springframework.ai.tool.definition.ToolDefinition; import org.springframework.ai.tool.execution.DefaultToolExecutionExceptionProcessor; import org.springframework.ai.tool.execution.ToolExecutionException; import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor; +import org.springframework.ai.tool.observation.DefaultToolCallingObservationConvention; +import org.springframework.ai.tool.observation.ToolCallingObservationContext; +import org.springframework.ai.tool.observation.ToolCallingObservationConvention; +import org.springframework.ai.tool.observation.ToolCallingObservationDocumentation; import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver; import org.springframework.ai.tool.resolution.ToolCallbackResolver; import org.springframework.util.Assert; @@ -58,6 +62,9 @@ public final class DefaultToolCallingManager implements ToolCallingManager { private static final ObservationRegistry DEFAULT_OBSERVATION_REGISTRY = ObservationRegistry.NOOP; + private static final ToolCallingObservationConvention DEFAULT_OBSERVATION_CONVENTION + = new DefaultToolCallingObservationConvention(); + private static final ToolCallbackResolver DEFAULT_TOOL_CALLBACK_RESOLVER = new DelegatingToolCallbackResolver(List.of()); @@ -72,6 +79,8 @@ public final class DefaultToolCallingManager implements ToolCallingManager { private final ToolExecutionExceptionProcessor toolExecutionExceptionProcessor; + private ToolCallingObservationConvention observationConvention = DEFAULT_OBSERVATION_CONVENTION; + public DefaultToolCallingManager(ObservationRegistry observationRegistry, ToolCallbackResolver toolCallbackResolver, ToolExecutionExceptionProcessor toolExecutionExceptionProcessor) { Assert.notNull(observationRegistry, "observationRegistry cannot be null"); @@ -104,7 +113,7 @@ public final class DefaultToolCallingManager implements ToolCallingManager { toolCallbacks.add(toolCallback); } - return toolCallbacks.stream().map(toolCallback -> toolCallback.getToolDefinition()).toList(); + return toolCallbacks.stream().map(ToolCallback::getToolDefinition).toList(); } @Override @@ -200,15 +209,29 @@ public final class DefaultToolCallingManager implements ToolCallingManager { returnDirect = returnDirect && toolCallback.getToolMetadata().returnDirect(); } - String toolResult; - try { - toolResult = toolCallback.call(toolInputArguments, toolContext); - } - catch (ToolExecutionException ex) { - toolResult = this.toolExecutionExceptionProcessor.process(ex); - } + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(toolCallback.getToolDefinition()) + .toolMetadata(toolCallback.getToolMetadata()) + .toolCallArguments(toolInputArguments) + .build(); - toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolName, toolResult)); + String toolCallResult = ToolCallingObservationDocumentation.TOOL_CALL + .observation(this.observationConvention, DEFAULT_OBSERVATION_CONVENTION, () -> observationContext, + this.observationRegistry) + .observe(() -> { + String toolResult; + try { + toolResult = toolCallback.call(toolInputArguments, toolContext); + } + catch (ToolExecutionException ex) { + toolResult = this.toolExecutionExceptionProcessor.process(ex); + } + observationContext.setToolCallResult(toolResult); + return toolResult; + }); + + toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolName, + toolCallResult != null ? toolCallResult : "")); } return new InternalToolExecutionResult(new ToolResponseMessage(toolResponses, Map.of()), returnDirect); @@ -222,6 +245,10 @@ public final class DefaultToolCallingManager implements ToolCallingManager { return messages; } + public void setObservationConvention(ToolCallingObservationConvention observationConvention) { + this.observationConvention = observationConvention; + } + public static Builder builder() { return new Builder(); } diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/definition/ToolDefinition.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/definition/ToolDefinition.java index 089c7b305..517a00617 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/tool/definition/ToolDefinition.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/definition/ToolDefinition.java @@ -39,4 +39,11 @@ public interface ToolDefinition { */ String inputSchema(); + /** + * Create a default {@link ToolDefinition} builder. + */ + static DefaultToolDefinition.Builder builder() { + return DefaultToolDefinition.builder(); + } + } diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConvention.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConvention.java new file mode 100644 index 000000000..0842ec5de --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConvention.java @@ -0,0 +1,106 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.common.KeyValue; +import io.micrometer.common.KeyValues; +import org.springframework.ai.observation.conventions.SpringAiKind; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Default conventions to populate observations for tool calling operations. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public class DefaultToolCallingObservationConvention implements ToolCallingObservationConvention { + + public static final String DEFAULT_NAME = "spring.ai.tool"; + + private final String name; + + public DefaultToolCallingObservationConvention() { + this(DEFAULT_NAME); + } + + public DefaultToolCallingObservationConvention(String name) { + this.name = name; + } + + @Override + public String getName() { + return this.name; + } + + @Override + @Nullable + public String getContextualName(ToolCallingObservationContext context) { + Assert.notNull(context, "context cannot be null"); + String toolName = context.getToolDefinition().name(); + return "%s %s".formatted(SpringAiKind.TOOL_CALL.value(), toolName); + } + + @Override + public KeyValues getLowCardinalityKeyValues(ToolCallingObservationContext context) { + return KeyValues.of(aiOperationType(context), aiProvider(context), springAiKind(context), + toolDefinitionName(context)); + } + + protected KeyValue aiOperationType(ToolCallingObservationContext context) { + return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE, + context.getOperationMetadata().operationType()); + } + + protected KeyValue aiProvider(ToolCallingObservationContext context) { + return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER, + context.getOperationMetadata().provider()); + } + + protected KeyValue springAiKind(ToolCallingObservationContext context) { + return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND, + SpringAiKind.TOOL_CALL.value()); + } + + protected KeyValue toolDefinitionName(ToolCallingObservationContext context) { + String toolName = context.getToolDefinition().name(); + return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME, toolName); + } + + @Override + public KeyValues getHighCardinalityKeyValues(ToolCallingObservationContext context) { + var keyValues = KeyValues.empty(); + keyValues = toolDefinitionDescription(keyValues, context); + keyValues = toolDefinitionSchema(keyValues, context); + return keyValues; + } + + protected KeyValues toolDefinitionDescription(KeyValues keyValues, ToolCallingObservationContext context) { + String toolDescription = context.getToolDefinition().description(); + return keyValues.and( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION.asString(), + toolDescription); + } + + protected KeyValues toolDefinitionSchema(KeyValues keyValues, ToolCallingObservationContext context) { + String toolSchema = context.getToolDefinition().inputSchema(); + return keyValues.and( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(), + toolSchema); + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilter.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilter.java new file mode 100644 index 000000000..9f1e5547e --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilter.java @@ -0,0 +1,52 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationFilter; + +/** + * An {@link ObservationFilter} to include the tool call content (input/output) in the + * observation. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public class ToolCallingContentObservationFilter implements ObservationFilter { + + @Override + public Observation.Context map(Observation.Context context) { + if (!(context instanceof ToolCallingObservationContext toolCallingObservationContext)) { + return context; + } + + String toolCallArguments = toolCallingObservationContext.getToolCallArguments(); + toolCallingObservationContext + .addHighCardinalityKeyValue(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS + .withValue(toolCallArguments)); + + String toolCallResult = toolCallingObservationContext.getToolCallResult(); + if (toolCallResult != null) { + toolCallingObservationContext + .addHighCardinalityKeyValue(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT + .withValue(toolCallResult)); + } + + return toolCallingObservationContext; + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationContext.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationContext.java new file mode 100644 index 000000000..3ce781824 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationContext.java @@ -0,0 +1,129 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.observation.Observation; +import org.springframework.ai.observation.AiOperationMetadata; +import org.springframework.ai.observation.conventions.AiOperationType; +import org.springframework.ai.observation.conventions.AiProvider; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Context used to store data for tool calling observations. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public class ToolCallingObservationContext extends Observation.Context { + + private final AiOperationMetadata operationMetadata = new AiOperationMetadata(AiOperationType.FRAMEWORK.value(), + AiProvider.SPRING_AI.value()); + + private final ToolDefinition toolDefinition; + + private final ToolMetadata toolMetadata; + + private final String toolCallArguments; + + @Nullable + private String toolCallResult; + + private ToolCallingObservationContext(ToolDefinition toolDefinition, ToolMetadata toolMetadata, + String toolCallArguments, @Nullable String toolCallResult) { + Assert.notNull(toolDefinition, "toolDefinition cannot be null"); + Assert.notNull(toolMetadata, "toolMetadata cannot be null"); + Assert.hasText(toolCallArguments, "toolCallArguments cannot be null or empty"); + + this.toolDefinition = toolDefinition; + this.toolMetadata = toolMetadata; + this.toolCallArguments = toolCallArguments; + this.toolCallResult = toolCallResult; + } + + public AiOperationMetadata getOperationMetadata() { + return operationMetadata; + } + + public ToolDefinition getToolDefinition() { + return toolDefinition; + } + + public ToolMetadata getToolMetadata() { + return toolMetadata; + } + + public String getToolCallArguments() { + return toolCallArguments; + } + + @Nullable + public String getToolCallResult() { + return toolCallResult; + } + + public void setToolCallResult(@Nullable String toolCallResult) { + this.toolCallResult = toolCallResult; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private ToolDefinition toolDefinition; + + private ToolMetadata toolMetadata = ToolMetadata.builder().build(); + + private String toolCallArguments; + + @Nullable + private String toolCallResult; + + private Builder() { + } + + public Builder toolDefinition(ToolDefinition toolDefinition) { + this.toolDefinition = toolDefinition; + return this; + } + + public Builder toolMetadata(ToolMetadata toolMetadata) { + this.toolMetadata = toolMetadata; + return this; + } + + public Builder toolCallArguments(String toolCallArguments) { + this.toolCallArguments = toolCallArguments; + return this; + } + + public Builder toolCallResult(@Nullable String toolCallResult) { + this.toolCallResult = toolCallResult; + return this; + } + + public ToolCallingObservationContext build() { + return new ToolCallingObservationContext(toolDefinition, toolMetadata, toolCallArguments, toolCallResult); + } + + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationConvention.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationConvention.java new file mode 100644 index 000000000..28481a9bb --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationConvention.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationConvention; + +/** + * Interface for an {@link ObservationConvention} for tool calling observations. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public interface ToolCallingObservationConvention extends ObservationConvention { + + @Override + default boolean supportsContext(Observation.Context context) { + return context instanceof ToolCallingObservationContext; + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationDocumentation.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationDocumentation.java new file mode 100644 index 000000000..dd6447c92 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/ToolCallingObservationDocumentation.java @@ -0,0 +1,148 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.common.docs.KeyName; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationConvention; +import io.micrometer.observation.docs.ObservationDocumentation; +import org.springframework.ai.observation.conventions.AiObservationAttributes; + +/** + * Tool calling observation documentation. + * + * @author Thomas Vitale + * @since 1.0.0 + */ +public enum ToolCallingObservationDocumentation implements ObservationDocumentation { + + /** + * Tool calling observations. + */ + TOOL_CALL { + @Override + public Class> getDefaultConvention() { + return DefaultToolCallingObservationConvention.class; + } + + @Override + public KeyName[] getLowCardinalityKeyNames() { + return LowCardinalityKeyNames.values(); + } + + @Override + public KeyName[] getHighCardinalityKeyNames() { + return HighCardinalityKeyNames.values(); + } + + }; + + /** + * Low cardinality key names. + */ + public enum LowCardinalityKeyNames implements KeyName { + + /** + * The name of the operation being performed. + */ + AI_OPERATION_TYPE { + @Override + public String asString() { + return AiObservationAttributes.AI_OPERATION_TYPE.value(); + } + }, + + /** + * The provider responsible for the operation. + */ + AI_PROVIDER { + @Override + public String asString() { + return AiObservationAttributes.AI_PROVIDER.value(); + } + }, + + /** + * Spring AI kind. + */ + SPRING_AI_KIND { + @Override + public String asString() { + return "spring.ai.kind"; + } + }, + + /** + * The name of the tool. + */ + TOOL_DEFINITION_NAME { + @Override + public String asString() { + return "spring.ai.tool.definition.name"; + } + }, + + } + + /** + * High cardinality key names. + */ + public enum HighCardinalityKeyNames implements KeyName { + + /** + * Description of the tool. + */ + TOOL_DEFINITION_DESCRIPTION { + @Override + public String asString() { + return "spring.ai.tool.definition.description"; + } + }, + + /** + * Schema of the parameters used to call the tool. + */ + TOOL_DEFINITION_SCHEMA { + @Override + public String asString() { + return "spring.ai.tool.definition.schema"; + } + }, + + /** + * The input arguments to the tool call. + */ + TOOL_CALL_ARGUMENTS { + @Override + public String asString() { + return "spring.ai.tool.call.arguments"; + } + }, + + /** + * The result of the tool call. + */ + TOOL_CALL_RESULT { + @Override + public String asString() { + return "spring.ai.tool.call.result"; + } + } + + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/package-info.java b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/package-info.java new file mode 100644 index 000000000..4708201ec --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/tool/observation/package-info.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides the API for chat client advisors observations. + */ +@NonNullApi +@NonNullFields +package org.springframework.ai.tool.observation; + +import org.springframework.lang.NonNullApi; +import org.springframework.lang.NonNullFields; diff --git a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java index 5629a1de4..da12a2490 100644 --- a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java +++ b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java @@ -32,6 +32,11 @@ import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.model.tool.ToolCallingChatOptions; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.ai.chat.observation.ChatModelObservationDocumentation.HighCardinalityKeyNames; @@ -145,6 +150,7 @@ class DefaultChatModelObservationConventionTests { HighCardinalityKeyNames.REQUEST_PRESENCE_PENALTY.asString(), HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES.asString(), HighCardinalityKeyNames.REQUEST_TEMPERATURE.asString(), + HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString(), HighCardinalityKeyNames.REQUEST_TOP_K.asString(), HighCardinalityKeyNames.REQUEST_TOP_P.asString(), HighCardinalityKeyNames.RESPONSE_FINISH_REASONS.asString(), HighCardinalityKeyNames.RESPONSE_ID.asString(), @@ -171,6 +177,23 @@ class DefaultChatModelObservationConventionTests { HighCardinalityKeyNames.RESPONSE_ID.asString()); } + @Test + void shouldHaveKeyValuesWhenTools() { + ChatModelObservationContext observationContext = ChatModelObservationContext.builder() + .prompt(generatePrompt(ToolCallingChatOptions.builder() + .model("mistral") + .toolNames("toolA", "toolB") + .toolCallbacks(new TestToolCallback("tool1", true), new TestToolCallback("tool2", false), + new TestToolCallback("toolB")) + .build())) + .provider("superprovider") + .build(); + assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)).anySatisfy(keyValue -> { + assertThat(keyValue.getKey()).isEqualTo(HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString()); + assertThat(keyValue.getValue()).contains("toolA", "toolB", "tool1", "tool2"); + }); + } + private Prompt generatePrompt(ChatOptions chatOptions) { return new Prompt("Who let the dogs out?", chatOptions); } @@ -198,4 +221,37 @@ class DefaultChatModelObservationConventionTests { } + static class TestToolCallback implements ToolCallback { + + private final ToolDefinition toolDefinition; + + private final ToolMetadata toolMetadata; + + TestToolCallback(String name) { + this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build(); + this.toolMetadata = ToolMetadata.builder().build(); + } + + TestToolCallback(String name, boolean returnDirect) { + this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build(); + this.toolMetadata = ToolMetadata.builder().returnDirect(returnDirect).build(); + } + + @Override + public ToolDefinition getToolDefinition() { + return this.toolDefinition; + } + + @Override + public ToolMetadata getToolMetadata() { + return this.toolMetadata; + } + + @Override + public String call(String toolInput) { + return "Mission accomplished!"; + } + + } + } diff --git a/spring-ai-model/src/test/java/org/springframework/ai/model/tool/DefaultToolCallingManagerIT.java b/spring-ai-model/src/test/java/org/springframework/ai/model/tool/DefaultToolCallingManagerIT.java new file mode 100644 index 000000000..e35d5a9a8 --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/model/tool/DefaultToolCallingManagerIT.java @@ -0,0 +1,159 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.model.tool; + +import io.micrometer.observation.tck.TestObservationRegistry; +import io.micrometer.observation.tck.TestObservationRegistryAssert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.metadata.ChatResponseMetadata; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.observation.conventions.AiOperationType; +import org.springframework.ai.observation.conventions.AiProvider; +import org.springframework.ai.observation.conventions.SpringAiKind; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; +import org.springframework.ai.tool.observation.DefaultToolCallingObservationConvention; +import org.springframework.ai.tool.observation.ToolCallingObservationDocumentation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests for {@link DefaultToolCallingManager}. + * + * @author Thomas Vitale + */ +@SpringBootTest(classes = DefaultToolCallingManagerIT.Config.class) +class DefaultToolCallingManagerIT { + + @Autowired + TestObservationRegistry observationRegistry; + + @Autowired + ToolCallingManager toolCallingManager; + + @BeforeEach + void beforeEach() { + this.observationRegistry.clear(); + } + + @Test + void observationForToolCall() { + ToolCallback toolCallback = new TestToolCallback("toolA"); + Prompt prompt = Prompt.builder() + .content("Why does a raven look like a desk?") + .chatOptions(ToolCallingChatOptions.builder().toolCallbacks(toolCallback).build()) + .build(); + + ChatResponse chatResponse = ChatResponse.builder() + .generations(List.of(new Generation(new AssistantMessage("Answer", Map.of(), + List.of(new AssistantMessage.ToolCall("toolA", "function", "toolA", "{}")))))) + .build(); + + ToolExecutionResult toolExecutionResult = toolCallingManager.executeToolCalls(prompt, chatResponse); + + assertThat(toolExecutionResult).isNotNull(); + + ChatResponseMetadata responseMetadata = chatResponse.getMetadata(); + assertThat(responseMetadata).isNotNull(); + + TestObservationRegistryAssert.assertThat(this.observationRegistry) + .doesNotHaveAnyRemainingCurrentObservation() + .hasObservationWithNameEqualTo(DefaultToolCallingObservationConvention.DEFAULT_NAME) + .that() + .hasContextualNameEqualTo(SpringAiKind.TOOL_CALL.value() + " " + toolCallback.getToolDefinition().name()) + .hasLowCardinalityKeyValue( + ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE.asString(), + AiOperationType.FRAMEWORK.value()) + .hasLowCardinalityKeyValue( + ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER.asString(), + AiProvider.SPRING_AI.value()) + .hasLowCardinalityKeyValue( + ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND.asString(), + SpringAiKind.TOOL_CALL.value()) + .hasLowCardinalityKeyValue( + ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME.asString(), + toolCallback.getToolDefinition().name()) + .hasHighCardinalityKeyValue( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION.asString(), + toolCallback.getToolDefinition().description()) + .hasHighCardinalityKeyValue( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(), + toolCallback.getToolDefinition().inputSchema()); + } + + @SpringBootConfiguration + static class Config { + + @Bean + public TestObservationRegistry observationRegistry() { + return TestObservationRegistry.create(); + } + + @Bean + public ToolCallingManager toolCallingManager(TestObservationRegistry observationRegistry) { + return DefaultToolCallingManager.builder().observationRegistry(observationRegistry).build(); + } + + } + + static class TestToolCallback implements ToolCallback { + + private final ToolDefinition toolDefinition; + + private final ToolMetadata toolMetadata; + + TestToolCallback(String name) { + this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build(); + this.toolMetadata = ToolMetadata.builder().build(); + } + + TestToolCallback(String name, boolean returnDirect) { + this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build(); + this.toolMetadata = ToolMetadata.builder().returnDirect(returnDirect).build(); + } + + @Override + public ToolDefinition getToolDefinition() { + return this.toolDefinition; + } + + @Override + public ToolMetadata getToolMetadata() { + return this.toolMetadata; + } + + @Override + public String call(String toolInput) { + return "Mission accomplished!"; + } + + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConventionTests.java b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConventionTests.java new file mode 100644 index 000000000..ebc71333b --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/DefaultToolCallingObservationConventionTests.java @@ -0,0 +1,100 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.common.KeyValue; +import io.micrometer.observation.Observation; +import org.junit.jupiter.api.Test; +import org.springframework.ai.observation.conventions.AiOperationType; +import org.springframework.ai.observation.conventions.AiProvider; +import org.springframework.ai.observation.conventions.SpringAiKind; +import org.springframework.ai.tool.definition.ToolDefinition; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link DefaultToolCallingObservationConvention}. + * + * @author Thomas Vitale + */ +class DefaultToolCallingObservationConventionTests { + + private final DefaultToolCallingObservationConvention observationConvention = new DefaultToolCallingObservationConvention(); + + @Test + void shouldHaveName() { + assertThat(this.observationConvention.getName()) + .isEqualTo(DefaultToolCallingObservationConvention.DEFAULT_NAME); + } + + @Test + void contextualName() { + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .build(); + assertThat(this.observationConvention.getContextualName(observationContext)).isEqualTo("tool_call toolA"); + } + + @Test + void supportsOnlyChatModelObservationContext() { + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .build(); + assertThat(this.observationConvention.supportsContext(observationContext)).isTrue(); + assertThat(this.observationConvention.supportsContext(new Observation.Context())).isFalse(); + } + + @Test + void shouldHaveLowCardinalityKeyValues() { + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .build(); + assertThat(this.observationConvention.getLowCardinalityKeyValues(observationContext)).contains( + KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME.asString(), + "toolA"), + KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE.asString(), + AiOperationType.FRAMEWORK.value()), + KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER.asString(), + AiProvider.SPRING_AI.value()), + KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND, + SpringAiKind.TOOL_CALL.value())); + } + + @Test + void shouldHaveHighCardinalityKeyValues() { + String toolCallInput = """ + { + "lizard": "George" + } + """; + ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments(toolCallInput) + .toolCallResult("Mission accomplished!") + .build(); + assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)).contains( + KeyValue.of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION + .asString(), "description"), + KeyValue.of( + ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(), + "{}")); + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilterTests.java b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilterTests.java new file mode 100644 index 000000000..b053946a7 --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingContentObservationFilterTests.java @@ -0,0 +1,76 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import io.micrometer.common.KeyValue; +import io.micrometer.observation.Observation; +import org.junit.jupiter.api.Test; +import org.springframework.ai.tool.definition.ToolDefinition; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link ToolCallingContentObservationFilter}. + * + * @author Thomas Vitale + */ +class ToolCallingContentObservationFilterTests { + + ToolCallingContentObservationFilter observationFilter = new ToolCallingContentObservationFilter(); + + @Test + void whenNotSupportedObservationContextThenReturnOriginalContext() { + var expectedContext = new Observation.Context(); + var actualContext = this.observationFilter.map(expectedContext); + + assertThat(actualContext).isEqualTo(expectedContext); + } + + @Test + void augmentContext() { + var originalContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .toolCallResult("result") + .build(); + var augmentedContext = this.observationFilter.map(originalContext); + + assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue + .of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS.asString(), "input")); + assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue + .of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT.asString(), "result")); + } + + @Test + void augmentContextWhenNullResult() { + var originalContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("input") + .toolCallResult("result") + .build(); + var augmentedContext = this.observationFilter.map(originalContext); + + assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue + .of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS.asString(), "input")); + assertThat(augmentedContext.getHighCardinalityKeyValues() + .stream() + .filter(kv -> kv.getKey() + .equals(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT.name()))) + .isEmpty(); + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingObservationContextTests.java b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingObservationContextTests.java new file mode 100644 index 000000000..cde5fc1fb --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/tool/observation/ToolCallingObservationContextTests.java @@ -0,0 +1,75 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.tool.observation; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.tool.definition.ToolDefinition; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Unit tests for {@link ToolCallingObservationContext}. + * + * @author Thomas Vitale + */ +class ToolCallingObservationContextTests { + + @Test + void whenMandatoryRequestOptionsThenReturn() { + var observationContext = ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("lizard") + .build(); + assertThat(observationContext).isNotNull(); + } + + @Test + void whenToolDefinitionIsNullThenThrow() { + assertThatThrownBy(() -> ToolCallingObservationContext.builder().toolCallArguments("lizard").build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("toolDefinition cannot be null"); + } + + @Test + void whenToolMetadataIsNullThenThrow() { + assertThatThrownBy(() -> ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("lizard") + .toolMetadata(null) + .build()).isInstanceOf(IllegalArgumentException.class).hasMessageContaining("toolMetadata cannot be null"); + } + + @Test + void whenToolCallInputIsNullThenThrow() { + assertThatThrownBy(() -> ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments(null) + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("toolCallArguments cannot be null or empty"); + } + + @Test + void whenToolCallInputIsEmptyThenThrow() { + assertThatThrownBy(() -> ToolCallingObservationContext.builder() + .toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build()) + .toolCallArguments("") + .build()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("toolCallArguments cannot be null or empty"); + } + +}