Micrometer instrumentation for tool calling
Introduce instrumentation for tool calling using the Micrometer Observation API. By default, metadata about tool calling are exported as metrics and traces. Optionally, the actual tool call input and result can be exported as well by enabling the dedicated feature flag. Signed-off-by: Thomas Vitale <ThomasVitale@users.noreply.github.com>
This commit is contained in:
committed by
Christian Tzolov
parent
8f879aae03
commit
30eb3ce3f2
@@ -16,17 +16,17 @@
|
||||
|
||||
package org.springframework.ai.model.tool.autoconfigure;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import io.micrometer.observation.ObservationRegistry;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.model.tool.ToolCallingManager;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.ToolCallbackProvider;
|
||||
import org.springframework.ai.tool.execution.DefaultToolExecutionExceptionProcessor;
|
||||
import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor;
|
||||
import org.springframework.ai.tool.observation.ToolCallingContentObservationFilter;
|
||||
import org.springframework.ai.tool.observation.ToolCallingObservationConvention;
|
||||
import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver;
|
||||
import org.springframework.ai.tool.resolution.SpringBeanToolCallbackResolver;
|
||||
import org.springframework.ai.tool.resolution.StaticToolCallbackResolver;
|
||||
@@ -35,9 +35,14 @@ import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.support.GenericApplicationContext;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Auto-configuration for common tool calling features of {@link ChatModel}.
|
||||
*
|
||||
@@ -47,8 +52,11 @@ import org.springframework.context.support.GenericApplicationContext;
|
||||
*/
|
||||
@AutoConfiguration
|
||||
@ConditionalOnClass(ChatModel.class)
|
||||
@EnableConfigurationProperties(ToolCallingProperties.class)
|
||||
public class ToolCallingAutoConfiguration {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ToolCallingAutoConfiguration.class);
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
ToolCallbackResolver toolCallbackResolver(GenericApplicationContext applicationContext,
|
||||
@@ -76,12 +84,27 @@ public class ToolCallingAutoConfiguration {
|
||||
@ConditionalOnMissingBean
|
||||
ToolCallingManager toolCallingManager(ToolCallbackResolver toolCallbackResolver,
|
||||
ToolExecutionExceptionProcessor toolExecutionExceptionProcessor,
|
||||
ObjectProvider<ObservationRegistry> observationRegistry) {
|
||||
return ToolCallingManager.builder()
|
||||
ObjectProvider<ObservationRegistry> observationRegistry,
|
||||
ObjectProvider<ToolCallingObservationConvention> observationConvention) {
|
||||
var toolCallingManager = ToolCallingManager.builder()
|
||||
.observationRegistry(observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP))
|
||||
.toolCallbackResolver(toolCallbackResolver)
|
||||
.toolExecutionExceptionProcessor(toolExecutionExceptionProcessor)
|
||||
.build();
|
||||
|
||||
observationConvention.ifAvailable(toolCallingManager::setObservationConvention);
|
||||
|
||||
return toolCallingManager;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
@ConditionalOnProperty(prefix = ToolCallingProperties.CONFIG_PREFIX + ".observations", name = "include-content",
|
||||
havingValue = "true")
|
||||
ToolCallingContentObservationFilter toolCallingContentObservationFilter() {
|
||||
logger.warn(
|
||||
"You have enabled the inclusion of the tool call arguments and result in the observations, with the risk of exposing sensitive or private information. Please, be careful!");
|
||||
return new ToolCallingContentObservationFilter();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.model.tool.autoconfigure;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
/**
|
||||
* Configuration properties for tool calling.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
* @since 1.0.0
|
||||
*/
|
||||
@ConfigurationProperties(ToolCallingProperties.CONFIG_PREFIX)
|
||||
public class ToolCallingProperties {
|
||||
|
||||
public static final String CONFIG_PREFIX = "spring.ai.tools";
|
||||
|
||||
private final Observations observations = new Observations();
|
||||
|
||||
public static class Observations {
|
||||
|
||||
/**
|
||||
* Whether to include the tool call content in the observations.
|
||||
*/
|
||||
private boolean includeContent = false;
|
||||
|
||||
public boolean isIncludeContent() {
|
||||
return includeContent;
|
||||
}
|
||||
|
||||
public void setIncludeContent(boolean includeContent) {
|
||||
this.includeContent = includeContent;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor;
|
||||
import org.springframework.ai.tool.function.FunctionToolCallback;
|
||||
import org.springframework.ai.tool.method.MethodToolCallback;
|
||||
import org.springframework.ai.tool.method.MethodToolCallbackProvider;
|
||||
import org.springframework.ai.tool.observation.ToolCallingContentObservationFilter;
|
||||
import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver;
|
||||
import org.springframework.ai.tool.resolution.ToolCallbackResolver;
|
||||
import org.springframework.ai.tool.support.ToolDefinitions;
|
||||
@@ -111,6 +112,25 @@ class ToolCallingAutoConfigurationTests {
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void observationFilterDefault() {
|
||||
new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(ToolCallingAutoConfiguration.class))
|
||||
.withUserConfiguration(Config.class)
|
||||
.run(context -> {
|
||||
assertThat(context).doesNotHaveBean(ToolCallingContentObservationFilter.class);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void observationFilterEnabled() {
|
||||
new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(ToolCallingAutoConfiguration.class))
|
||||
.withPropertyValues("spring.ai.tools.observations.include-content=true")
|
||||
.withUserConfiguration(Config.class)
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(ToolCallingContentObservationFilter.class);
|
||||
});
|
||||
}
|
||||
|
||||
static class WeatherService {
|
||||
|
||||
@Tool(description = "Get the weather in location. Return temperature in 36°F or 36°C format.")
|
||||
|
||||
@@ -67,6 +67,10 @@ public enum AiObservationAttributes {
|
||||
* The temperature setting for the model request.
|
||||
*/
|
||||
REQUEST_TEMPERATURE("gen_ai.request.temperature"),
|
||||
/**
|
||||
* List of tool definitions provided to the model in the request.
|
||||
*/
|
||||
REQUEST_TOOL_NAMES("spring.ai.model.request.tool.names"),
|
||||
/**
|
||||
* The top_k sampling setting for the model request.
|
||||
*/
|
||||
|
||||
@@ -37,6 +37,11 @@ public enum SpringAiKind {
|
||||
*/
|
||||
CHAT_CLIENT("chat_client"),
|
||||
|
||||
/**
|
||||
* Spring AI kind for tool calling.
|
||||
*/
|
||||
TOOL_CALL("tool_call"),
|
||||
|
||||
/**
|
||||
* Spring AI kind for vector store.
|
||||
*/
|
||||
|
||||
@@ -155,6 +155,7 @@ IMPORTANT: The `gen_ai.client.token.usage` metrics measures number of input and
|
||||
|`gen_ai.usage.total_tokens` | The total number of tokens used in the model exchange.
|
||||
|`gen_ai.prompt` | The full prompt sent to the model. Optional.
|
||||
|`gen_ai.completion` | The full response received from the model. Optional.
|
||||
|`spring.ai.model.request.tool.names` | List of tool definitions provided to the model in the request.
|
||||
|===
|
||||
|
||||
NOTE: For measuring user tokens, the previous table lists the values present in an observation trace.
|
||||
@@ -179,6 +180,46 @@ Spring AI supports logging chat prompt and completion data, useful for troublesh
|
||||
|
||||
WARNING: If you enable logging of the chat prompt and completion data, there's a risk of exposing sensitive or private information. Please, be careful!
|
||||
|
||||
== Tool Calling
|
||||
|
||||
The `spring.ai.tool` observations are recorded when performing tool calling in the context of a chat model interaction. They measure the time spent on toll call completion and propagate the related tracing information.
|
||||
|
||||
.Low Cardinality Keys
|
||||
[cols="a,a", stripes=even]
|
||||
|===
|
||||
|Name | Description
|
||||
|
||||
|`gen_ai.operation.name` | The name of the operation being performed. It's always `framework`.
|
||||
|`gen_ai.system` | The provider responsible for the operation. It's always `spring_ai`.
|
||||
|`spring.ai.kind` | The kind of operation performed by Spring AI. It's always `tool_call`.
|
||||
|`spring.ai.tool.definition.name` | The name of the tool.
|
||||
|===
|
||||
|
||||
.High Cardinality Keys
|
||||
[cols="a,a", stripes=even]
|
||||
|===
|
||||
|Name | Description
|
||||
|`spring.ai.tool.definition.description` | Description of the tool.
|
||||
|`spring.ai.tool.definition.schema` | Schema of the parameters used to call the tool.
|
||||
|`spring.ai.tool.call.arguments` | The input arguments to the tool call. (Only when enabled)
|
||||
|`spring.ai.tool.call.result` | Schema of the parameters used to call the tool. (Only when enabled)
|
||||
|===
|
||||
|
||||
=== Tool Call Arguments and Result Data
|
||||
|
||||
The input arguments and result from the tool call are not exported by default, as they can be potentially sensitive.
|
||||
|
||||
Spring AI supports exporting tool call arguments and result data as span attributes.
|
||||
|
||||
[cols="6,3,1", stripes=even]
|
||||
|====
|
||||
| Property | Description | Default
|
||||
|
||||
| `spring.ai.tools.observations.include-content` | Include the tool call content in observations. `true` or `false` | `false`
|
||||
|====
|
||||
|
||||
WARNING: If you enable the inclusion of the tool call arguments and result in the observations, there's a risk of exposing sensitive or private information. Please, be careful!
|
||||
|
||||
== EmbeddingModel
|
||||
|
||||
NOTE: Observability features are currently supported only for `EmbeddingModel` implementations from the following
|
||||
|
||||
@@ -150,6 +150,16 @@ public enum ChatModelObservationDocumentation implements ObservationDocumentatio
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* List of tool definitions provided to the model in the request.
|
||||
*/
|
||||
REQUEST_TOOL_NAMES {
|
||||
@Override
|
||||
public String asString() {
|
||||
return AiObservationAttributes.REQUEST_TOOL_NAMES.value();
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* The top_k sampling setting for the model request.
|
||||
*/
|
||||
|
||||
@@ -16,13 +16,15 @@
|
||||
|
||||
package org.springframework.ai.chat.observation;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
import io.micrometer.common.KeyValue;
|
||||
import io.micrometer.common.KeyValues;
|
||||
|
||||
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||
import org.springframework.ai.model.tool.ToolCallingChatOptions;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
@@ -100,6 +102,7 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati
|
||||
keyValues = requestPresencePenalty(keyValues, context);
|
||||
keyValues = requestStopSequences(keyValues, context);
|
||||
keyValues = requestTemperature(keyValues, context);
|
||||
keyValues = requestTools(keyValues, context);
|
||||
keyValues = requestTopK(keyValues, context);
|
||||
keyValues = requestTopP(keyValues, context);
|
||||
// Response
|
||||
@@ -148,8 +151,6 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati
|
||||
if (!CollectionUtils.isEmpty(options.getStopSequences())) {
|
||||
StringJoiner stopSequencesJoiner = new StringJoiner(", ", "[", "]");
|
||||
options.getStopSequences().forEach(value -> stopSequencesJoiner.add("\"" + value + "\""));
|
||||
KeyValue.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES,
|
||||
options.getStopSequences(), Objects::nonNull);
|
||||
return keyValues.and(
|
||||
ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES.asString(),
|
||||
stopSequencesJoiner.toString());
|
||||
@@ -167,6 +168,24 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati
|
||||
return keyValues;
|
||||
}
|
||||
|
||||
protected KeyValues requestTools(KeyValues keyValues, ChatModelObservationContext context) {
|
||||
if (!(context.getRequest().getOptions() instanceof ToolCallingChatOptions options)) {
|
||||
return keyValues;
|
||||
}
|
||||
|
||||
Set<String> toolNames = new HashSet<>(options.getToolNames());
|
||||
toolNames.addAll(options.getToolCallbacks().stream().map(tc -> tc.getToolDefinition().name()).toList());
|
||||
|
||||
if (!CollectionUtils.isEmpty(toolNames)) {
|
||||
StringJoiner toolNamesJoiner = new StringJoiner(", ", "[", "]");
|
||||
toolNames.forEach(value -> toolNamesJoiner.add("\"" + value + "\""));
|
||||
return keyValues.and(
|
||||
ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString(),
|
||||
toolNamesJoiner.toString());
|
||||
}
|
||||
return keyValues;
|
||||
}
|
||||
|
||||
protected KeyValues requestTopK(KeyValues keyValues, ChatModelObservationContext context) {
|
||||
ChatOptions options = context.getRequest().getOptions();
|
||||
if (options.getTopK() != null) {
|
||||
|
||||
@@ -38,6 +38,10 @@ import org.springframework.ai.tool.definition.ToolDefinition;
|
||||
import org.springframework.ai.tool.execution.DefaultToolExecutionExceptionProcessor;
|
||||
import org.springframework.ai.tool.execution.ToolExecutionException;
|
||||
import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor;
|
||||
import org.springframework.ai.tool.observation.DefaultToolCallingObservationConvention;
|
||||
import org.springframework.ai.tool.observation.ToolCallingObservationContext;
|
||||
import org.springframework.ai.tool.observation.ToolCallingObservationConvention;
|
||||
import org.springframework.ai.tool.observation.ToolCallingObservationDocumentation;
|
||||
import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver;
|
||||
import org.springframework.ai.tool.resolution.ToolCallbackResolver;
|
||||
import org.springframework.util.Assert;
|
||||
@@ -58,6 +62,9 @@ public final class DefaultToolCallingManager implements ToolCallingManager {
|
||||
private static final ObservationRegistry DEFAULT_OBSERVATION_REGISTRY
|
||||
= ObservationRegistry.NOOP;
|
||||
|
||||
private static final ToolCallingObservationConvention DEFAULT_OBSERVATION_CONVENTION
|
||||
= new DefaultToolCallingObservationConvention();
|
||||
|
||||
private static final ToolCallbackResolver DEFAULT_TOOL_CALLBACK_RESOLVER
|
||||
= new DelegatingToolCallbackResolver(List.of());
|
||||
|
||||
@@ -72,6 +79,8 @@ public final class DefaultToolCallingManager implements ToolCallingManager {
|
||||
|
||||
private final ToolExecutionExceptionProcessor toolExecutionExceptionProcessor;
|
||||
|
||||
private ToolCallingObservationConvention observationConvention = DEFAULT_OBSERVATION_CONVENTION;
|
||||
|
||||
public DefaultToolCallingManager(ObservationRegistry observationRegistry, ToolCallbackResolver toolCallbackResolver,
|
||||
ToolExecutionExceptionProcessor toolExecutionExceptionProcessor) {
|
||||
Assert.notNull(observationRegistry, "observationRegistry cannot be null");
|
||||
@@ -104,7 +113,7 @@ public final class DefaultToolCallingManager implements ToolCallingManager {
|
||||
toolCallbacks.add(toolCallback);
|
||||
}
|
||||
|
||||
return toolCallbacks.stream().map(toolCallback -> toolCallback.getToolDefinition()).toList();
|
||||
return toolCallbacks.stream().map(ToolCallback::getToolDefinition).toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -200,15 +209,29 @@ public final class DefaultToolCallingManager implements ToolCallingManager {
|
||||
returnDirect = returnDirect && toolCallback.getToolMetadata().returnDirect();
|
||||
}
|
||||
|
||||
String toolResult;
|
||||
try {
|
||||
toolResult = toolCallback.call(toolInputArguments, toolContext);
|
||||
}
|
||||
catch (ToolExecutionException ex) {
|
||||
toolResult = this.toolExecutionExceptionProcessor.process(ex);
|
||||
}
|
||||
ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder()
|
||||
.toolDefinition(toolCallback.getToolDefinition())
|
||||
.toolMetadata(toolCallback.getToolMetadata())
|
||||
.toolCallArguments(toolInputArguments)
|
||||
.build();
|
||||
|
||||
toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolName, toolResult));
|
||||
String toolCallResult = ToolCallingObservationDocumentation.TOOL_CALL
|
||||
.observation(this.observationConvention, DEFAULT_OBSERVATION_CONVENTION, () -> observationContext,
|
||||
this.observationRegistry)
|
||||
.observe(() -> {
|
||||
String toolResult;
|
||||
try {
|
||||
toolResult = toolCallback.call(toolInputArguments, toolContext);
|
||||
}
|
||||
catch (ToolExecutionException ex) {
|
||||
toolResult = this.toolExecutionExceptionProcessor.process(ex);
|
||||
}
|
||||
observationContext.setToolCallResult(toolResult);
|
||||
return toolResult;
|
||||
});
|
||||
|
||||
toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolName,
|
||||
toolCallResult != null ? toolCallResult : ""));
|
||||
}
|
||||
|
||||
return new InternalToolExecutionResult(new ToolResponseMessage(toolResponses, Map.of()), returnDirect);
|
||||
@@ -222,6 +245,10 @@ public final class DefaultToolCallingManager implements ToolCallingManager {
|
||||
return messages;
|
||||
}
|
||||
|
||||
public void setObservationConvention(ToolCallingObservationConvention observationConvention) {
|
||||
this.observationConvention = observationConvention;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
@@ -39,4 +39,11 @@ public interface ToolDefinition {
|
||||
*/
|
||||
String inputSchema();
|
||||
|
||||
/**
|
||||
* Create a default {@link ToolDefinition} builder.
|
||||
*/
|
||||
static DefaultToolDefinition.Builder builder() {
|
||||
return DefaultToolDefinition.builder();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import io.micrometer.common.KeyValue;
|
||||
import io.micrometer.common.KeyValues;
|
||||
import org.springframework.ai.observation.conventions.SpringAiKind;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* Default conventions to populate observations for tool calling operations.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public class DefaultToolCallingObservationConvention implements ToolCallingObservationConvention {
|
||||
|
||||
public static final String DEFAULT_NAME = "spring.ai.tool";
|
||||
|
||||
private final String name;
|
||||
|
||||
public DefaultToolCallingObservationConvention() {
|
||||
this(DEFAULT_NAME);
|
||||
}
|
||||
|
||||
public DefaultToolCallingObservationConvention(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Nullable
|
||||
public String getContextualName(ToolCallingObservationContext context) {
|
||||
Assert.notNull(context, "context cannot be null");
|
||||
String toolName = context.getToolDefinition().name();
|
||||
return "%s %s".formatted(SpringAiKind.TOOL_CALL.value(), toolName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KeyValues getLowCardinalityKeyValues(ToolCallingObservationContext context) {
|
||||
return KeyValues.of(aiOperationType(context), aiProvider(context), springAiKind(context),
|
||||
toolDefinitionName(context));
|
||||
}
|
||||
|
||||
protected KeyValue aiOperationType(ToolCallingObservationContext context) {
|
||||
return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE,
|
||||
context.getOperationMetadata().operationType());
|
||||
}
|
||||
|
||||
protected KeyValue aiProvider(ToolCallingObservationContext context) {
|
||||
return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER,
|
||||
context.getOperationMetadata().provider());
|
||||
}
|
||||
|
||||
protected KeyValue springAiKind(ToolCallingObservationContext context) {
|
||||
return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND,
|
||||
SpringAiKind.TOOL_CALL.value());
|
||||
}
|
||||
|
||||
protected KeyValue toolDefinitionName(ToolCallingObservationContext context) {
|
||||
String toolName = context.getToolDefinition().name();
|
||||
return KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME, toolName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KeyValues getHighCardinalityKeyValues(ToolCallingObservationContext context) {
|
||||
var keyValues = KeyValues.empty();
|
||||
keyValues = toolDefinitionDescription(keyValues, context);
|
||||
keyValues = toolDefinitionSchema(keyValues, context);
|
||||
return keyValues;
|
||||
}
|
||||
|
||||
protected KeyValues toolDefinitionDescription(KeyValues keyValues, ToolCallingObservationContext context) {
|
||||
String toolDescription = context.getToolDefinition().description();
|
||||
return keyValues.and(
|
||||
ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION.asString(),
|
||||
toolDescription);
|
||||
}
|
||||
|
||||
protected KeyValues toolDefinitionSchema(KeyValues keyValues, ToolCallingObservationContext context) {
|
||||
String toolSchema = context.getToolDefinition().inputSchema();
|
||||
return keyValues.and(
|
||||
ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(),
|
||||
toolSchema);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import io.micrometer.observation.Observation;
|
||||
import io.micrometer.observation.ObservationFilter;
|
||||
|
||||
/**
|
||||
* An {@link ObservationFilter} to include the tool call content (input/output) in the
|
||||
* observation.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public class ToolCallingContentObservationFilter implements ObservationFilter {
|
||||
|
||||
@Override
|
||||
public Observation.Context map(Observation.Context context) {
|
||||
if (!(context instanceof ToolCallingObservationContext toolCallingObservationContext)) {
|
||||
return context;
|
||||
}
|
||||
|
||||
String toolCallArguments = toolCallingObservationContext.getToolCallArguments();
|
||||
toolCallingObservationContext
|
||||
.addHighCardinalityKeyValue(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS
|
||||
.withValue(toolCallArguments));
|
||||
|
||||
String toolCallResult = toolCallingObservationContext.getToolCallResult();
|
||||
if (toolCallResult != null) {
|
||||
toolCallingObservationContext
|
||||
.addHighCardinalityKeyValue(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT
|
||||
.withValue(toolCallResult));
|
||||
}
|
||||
|
||||
return toolCallingObservationContext;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import io.micrometer.observation.Observation;
|
||||
import org.springframework.ai.observation.AiOperationMetadata;
|
||||
import org.springframework.ai.observation.conventions.AiOperationType;
|
||||
import org.springframework.ai.observation.conventions.AiProvider;
|
||||
import org.springframework.ai.tool.definition.ToolDefinition;
|
||||
import org.springframework.ai.tool.metadata.ToolMetadata;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* Context used to store data for tool calling observations.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public class ToolCallingObservationContext extends Observation.Context {
|
||||
|
||||
private final AiOperationMetadata operationMetadata = new AiOperationMetadata(AiOperationType.FRAMEWORK.value(),
|
||||
AiProvider.SPRING_AI.value());
|
||||
|
||||
private final ToolDefinition toolDefinition;
|
||||
|
||||
private final ToolMetadata toolMetadata;
|
||||
|
||||
private final String toolCallArguments;
|
||||
|
||||
@Nullable
|
||||
private String toolCallResult;
|
||||
|
||||
private ToolCallingObservationContext(ToolDefinition toolDefinition, ToolMetadata toolMetadata,
|
||||
String toolCallArguments, @Nullable String toolCallResult) {
|
||||
Assert.notNull(toolDefinition, "toolDefinition cannot be null");
|
||||
Assert.notNull(toolMetadata, "toolMetadata cannot be null");
|
||||
Assert.hasText(toolCallArguments, "toolCallArguments cannot be null or empty");
|
||||
|
||||
this.toolDefinition = toolDefinition;
|
||||
this.toolMetadata = toolMetadata;
|
||||
this.toolCallArguments = toolCallArguments;
|
||||
this.toolCallResult = toolCallResult;
|
||||
}
|
||||
|
||||
public AiOperationMetadata getOperationMetadata() {
|
||||
return operationMetadata;
|
||||
}
|
||||
|
||||
public ToolDefinition getToolDefinition() {
|
||||
return toolDefinition;
|
||||
}
|
||||
|
||||
public ToolMetadata getToolMetadata() {
|
||||
return toolMetadata;
|
||||
}
|
||||
|
||||
public String getToolCallArguments() {
|
||||
return toolCallArguments;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public String getToolCallResult() {
|
||||
return toolCallResult;
|
||||
}
|
||||
|
||||
public void setToolCallResult(@Nullable String toolCallResult) {
|
||||
this.toolCallResult = toolCallResult;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
|
||||
private ToolDefinition toolDefinition;
|
||||
|
||||
private ToolMetadata toolMetadata = ToolMetadata.builder().build();
|
||||
|
||||
private String toolCallArguments;
|
||||
|
||||
@Nullable
|
||||
private String toolCallResult;
|
||||
|
||||
private Builder() {
|
||||
}
|
||||
|
||||
public Builder toolDefinition(ToolDefinition toolDefinition) {
|
||||
this.toolDefinition = toolDefinition;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder toolMetadata(ToolMetadata toolMetadata) {
|
||||
this.toolMetadata = toolMetadata;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder toolCallArguments(String toolCallArguments) {
|
||||
this.toolCallArguments = toolCallArguments;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder toolCallResult(@Nullable String toolCallResult) {
|
||||
this.toolCallResult = toolCallResult;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ToolCallingObservationContext build() {
|
||||
return new ToolCallingObservationContext(toolDefinition, toolMetadata, toolCallArguments, toolCallResult);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import io.micrometer.observation.Observation;
|
||||
import io.micrometer.observation.ObservationConvention;
|
||||
|
||||
/**
|
||||
* Interface for an {@link ObservationConvention} for tool calling observations.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public interface ToolCallingObservationConvention extends ObservationConvention<ToolCallingObservationContext> {
|
||||
|
||||
@Override
|
||||
default boolean supportsContext(Observation.Context context) {
|
||||
return context instanceof ToolCallingObservationContext;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import io.micrometer.common.docs.KeyName;
|
||||
import io.micrometer.observation.Observation;
|
||||
import io.micrometer.observation.ObservationConvention;
|
||||
import io.micrometer.observation.docs.ObservationDocumentation;
|
||||
import org.springframework.ai.observation.conventions.AiObservationAttributes;
|
||||
|
||||
/**
|
||||
* Tool calling observation documentation.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public enum ToolCallingObservationDocumentation implements ObservationDocumentation {
|
||||
|
||||
/**
|
||||
* Tool calling observations.
|
||||
*/
|
||||
TOOL_CALL {
|
||||
@Override
|
||||
public Class<? extends ObservationConvention<? extends Observation.Context>> getDefaultConvention() {
|
||||
return DefaultToolCallingObservationConvention.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
public KeyName[] getLowCardinalityKeyNames() {
|
||||
return LowCardinalityKeyNames.values();
|
||||
}
|
||||
|
||||
@Override
|
||||
public KeyName[] getHighCardinalityKeyNames() {
|
||||
return HighCardinalityKeyNames.values();
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* Low cardinality key names.
|
||||
*/
|
||||
public enum LowCardinalityKeyNames implements KeyName {
|
||||
|
||||
/**
|
||||
* The name of the operation being performed.
|
||||
*/
|
||||
AI_OPERATION_TYPE {
|
||||
@Override
|
||||
public String asString() {
|
||||
return AiObservationAttributes.AI_OPERATION_TYPE.value();
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* The provider responsible for the operation.
|
||||
*/
|
||||
AI_PROVIDER {
|
||||
@Override
|
||||
public String asString() {
|
||||
return AiObservationAttributes.AI_PROVIDER.value();
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Spring AI kind.
|
||||
*/
|
||||
SPRING_AI_KIND {
|
||||
@Override
|
||||
public String asString() {
|
||||
return "spring.ai.kind";
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* The name of the tool.
|
||||
*/
|
||||
TOOL_DEFINITION_NAME {
|
||||
@Override
|
||||
public String asString() {
|
||||
return "spring.ai.tool.definition.name";
|
||||
}
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* High cardinality key names.
|
||||
*/
|
||||
public enum HighCardinalityKeyNames implements KeyName {
|
||||
|
||||
/**
|
||||
* Description of the tool.
|
||||
*/
|
||||
TOOL_DEFINITION_DESCRIPTION {
|
||||
@Override
|
||||
public String asString() {
|
||||
return "spring.ai.tool.definition.description";
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Schema of the parameters used to call the tool.
|
||||
*/
|
||||
TOOL_DEFINITION_SCHEMA {
|
||||
@Override
|
||||
public String asString() {
|
||||
return "spring.ai.tool.definition.schema";
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* The input arguments to the tool call.
|
||||
*/
|
||||
TOOL_CALL_ARGUMENTS {
|
||||
@Override
|
||||
public String asString() {
|
||||
return "spring.ai.tool.call.arguments";
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* The result of the tool call.
|
||||
*/
|
||||
TOOL_CALL_RESULT {
|
||||
@Override
|
||||
public String asString() {
|
||||
return "spring.ai.tool.call.result";
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright 2023-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Provides the API for chat client advisors observations.
|
||||
*/
|
||||
@NonNullApi
|
||||
@NonNullFields
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import org.springframework.lang.NonNullApi;
|
||||
import org.springframework.lang.NonNullFields;
|
||||
@@ -32,6 +32,11 @@ import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.model.tool.ToolCallingChatOptions;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.definition.DefaultToolDefinition;
|
||||
import org.springframework.ai.tool.definition.ToolDefinition;
|
||||
import org.springframework.ai.tool.metadata.ToolMetadata;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.springframework.ai.chat.observation.ChatModelObservationDocumentation.HighCardinalityKeyNames;
|
||||
@@ -145,6 +150,7 @@ class DefaultChatModelObservationConventionTests {
|
||||
HighCardinalityKeyNames.REQUEST_PRESENCE_PENALTY.asString(),
|
||||
HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES.asString(),
|
||||
HighCardinalityKeyNames.REQUEST_TEMPERATURE.asString(),
|
||||
HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString(),
|
||||
HighCardinalityKeyNames.REQUEST_TOP_K.asString(), HighCardinalityKeyNames.REQUEST_TOP_P.asString(),
|
||||
HighCardinalityKeyNames.RESPONSE_FINISH_REASONS.asString(),
|
||||
HighCardinalityKeyNames.RESPONSE_ID.asString(),
|
||||
@@ -171,6 +177,23 @@ class DefaultChatModelObservationConventionTests {
|
||||
HighCardinalityKeyNames.RESPONSE_ID.asString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldHaveKeyValuesWhenTools() {
|
||||
ChatModelObservationContext observationContext = ChatModelObservationContext.builder()
|
||||
.prompt(generatePrompt(ToolCallingChatOptions.builder()
|
||||
.model("mistral")
|
||||
.toolNames("toolA", "toolB")
|
||||
.toolCallbacks(new TestToolCallback("tool1", true), new TestToolCallback("tool2", false),
|
||||
new TestToolCallback("toolB"))
|
||||
.build()))
|
||||
.provider("superprovider")
|
||||
.build();
|
||||
assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)).anySatisfy(keyValue -> {
|
||||
assertThat(keyValue.getKey()).isEqualTo(HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString());
|
||||
assertThat(keyValue.getValue()).contains("toolA", "toolB", "tool1", "tool2");
|
||||
});
|
||||
}
|
||||
|
||||
private Prompt generatePrompt(ChatOptions chatOptions) {
|
||||
return new Prompt("Who let the dogs out?", chatOptions);
|
||||
}
|
||||
@@ -198,4 +221,37 @@ class DefaultChatModelObservationConventionTests {
|
||||
|
||||
}
|
||||
|
||||
static class TestToolCallback implements ToolCallback {
|
||||
|
||||
private final ToolDefinition toolDefinition;
|
||||
|
||||
private final ToolMetadata toolMetadata;
|
||||
|
||||
TestToolCallback(String name) {
|
||||
this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build();
|
||||
this.toolMetadata = ToolMetadata.builder().build();
|
||||
}
|
||||
|
||||
TestToolCallback(String name, boolean returnDirect) {
|
||||
this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build();
|
||||
this.toolMetadata = ToolMetadata.builder().returnDirect(returnDirect).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ToolDefinition getToolDefinition() {
|
||||
return this.toolDefinition;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ToolMetadata getToolMetadata() {
|
||||
return this.toolMetadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String call(String toolInput) {
|
||||
return "Mission accomplished!";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.model.tool;
|
||||
|
||||
import io.micrometer.observation.tck.TestObservationRegistry;
|
||||
import io.micrometer.observation.tck.TestObservationRegistryAssert;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.observation.conventions.AiOperationType;
|
||||
import org.springframework.ai.observation.conventions.AiProvider;
|
||||
import org.springframework.ai.observation.conventions.SpringAiKind;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.definition.DefaultToolDefinition;
|
||||
import org.springframework.ai.tool.definition.ToolDefinition;
|
||||
import org.springframework.ai.tool.metadata.ToolMetadata;
|
||||
import org.springframework.ai.tool.observation.DefaultToolCallingObservationConvention;
|
||||
import org.springframework.ai.tool.observation.ToolCallingObservationDocumentation;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringBootConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Integration tests for {@link DefaultToolCallingManager}.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
*/
|
||||
@SpringBootTest(classes = DefaultToolCallingManagerIT.Config.class)
|
||||
class DefaultToolCallingManagerIT {
|
||||
|
||||
@Autowired
|
||||
TestObservationRegistry observationRegistry;
|
||||
|
||||
@Autowired
|
||||
ToolCallingManager toolCallingManager;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
this.observationRegistry.clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
void observationForToolCall() {
|
||||
ToolCallback toolCallback = new TestToolCallback("toolA");
|
||||
Prompt prompt = Prompt.builder()
|
||||
.content("Why does a raven look like a desk?")
|
||||
.chatOptions(ToolCallingChatOptions.builder().toolCallbacks(toolCallback).build())
|
||||
.build();
|
||||
|
||||
ChatResponse chatResponse = ChatResponse.builder()
|
||||
.generations(List.of(new Generation(new AssistantMessage("Answer", Map.of(),
|
||||
List.of(new AssistantMessage.ToolCall("toolA", "function", "toolA", "{}"))))))
|
||||
.build();
|
||||
|
||||
ToolExecutionResult toolExecutionResult = toolCallingManager.executeToolCalls(prompt, chatResponse);
|
||||
|
||||
assertThat(toolExecutionResult).isNotNull();
|
||||
|
||||
ChatResponseMetadata responseMetadata = chatResponse.getMetadata();
|
||||
assertThat(responseMetadata).isNotNull();
|
||||
|
||||
TestObservationRegistryAssert.assertThat(this.observationRegistry)
|
||||
.doesNotHaveAnyRemainingCurrentObservation()
|
||||
.hasObservationWithNameEqualTo(DefaultToolCallingObservationConvention.DEFAULT_NAME)
|
||||
.that()
|
||||
.hasContextualNameEqualTo(SpringAiKind.TOOL_CALL.value() + " " + toolCallback.getToolDefinition().name())
|
||||
.hasLowCardinalityKeyValue(
|
||||
ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE.asString(),
|
||||
AiOperationType.FRAMEWORK.value())
|
||||
.hasLowCardinalityKeyValue(
|
||||
ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER.asString(),
|
||||
AiProvider.SPRING_AI.value())
|
||||
.hasLowCardinalityKeyValue(
|
||||
ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND.asString(),
|
||||
SpringAiKind.TOOL_CALL.value())
|
||||
.hasLowCardinalityKeyValue(
|
||||
ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME.asString(),
|
||||
toolCallback.getToolDefinition().name())
|
||||
.hasHighCardinalityKeyValue(
|
||||
ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION.asString(),
|
||||
toolCallback.getToolDefinition().description())
|
||||
.hasHighCardinalityKeyValue(
|
||||
ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(),
|
||||
toolCallback.getToolDefinition().inputSchema());
|
||||
}
|
||||
|
||||
@SpringBootConfiguration
|
||||
static class Config {
|
||||
|
||||
@Bean
|
||||
public TestObservationRegistry observationRegistry() {
|
||||
return TestObservationRegistry.create();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ToolCallingManager toolCallingManager(TestObservationRegistry observationRegistry) {
|
||||
return DefaultToolCallingManager.builder().observationRegistry(observationRegistry).build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static class TestToolCallback implements ToolCallback {
|
||||
|
||||
private final ToolDefinition toolDefinition;
|
||||
|
||||
private final ToolMetadata toolMetadata;
|
||||
|
||||
TestToolCallback(String name) {
|
||||
this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build();
|
||||
this.toolMetadata = ToolMetadata.builder().build();
|
||||
}
|
||||
|
||||
TestToolCallback(String name, boolean returnDirect) {
|
||||
this.toolDefinition = DefaultToolDefinition.builder().name(name).inputSchema("{}").build();
|
||||
this.toolMetadata = ToolMetadata.builder().returnDirect(returnDirect).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ToolDefinition getToolDefinition() {
|
||||
return this.toolDefinition;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ToolMetadata getToolMetadata() {
|
||||
return this.toolMetadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String call(String toolInput) {
|
||||
return "Mission accomplished!";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import io.micrometer.common.KeyValue;
|
||||
import io.micrometer.observation.Observation;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.observation.conventions.AiOperationType;
|
||||
import org.springframework.ai.observation.conventions.AiProvider;
|
||||
import org.springframework.ai.observation.conventions.SpringAiKind;
|
||||
import org.springframework.ai.tool.definition.ToolDefinition;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link DefaultToolCallingObservationConvention}.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
*/
|
||||
class DefaultToolCallingObservationConventionTests {
|
||||
|
||||
private final DefaultToolCallingObservationConvention observationConvention = new DefaultToolCallingObservationConvention();
|
||||
|
||||
@Test
|
||||
void shouldHaveName() {
|
||||
assertThat(this.observationConvention.getName())
|
||||
.isEqualTo(DefaultToolCallingObservationConvention.DEFAULT_NAME);
|
||||
}
|
||||
|
||||
@Test
|
||||
void contextualName() {
|
||||
ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments("input")
|
||||
.build();
|
||||
assertThat(this.observationConvention.getContextualName(observationContext)).isEqualTo("tool_call toolA");
|
||||
}
|
||||
|
||||
@Test
|
||||
void supportsOnlyChatModelObservationContext() {
|
||||
ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments("input")
|
||||
.build();
|
||||
assertThat(this.observationConvention.supportsContext(observationContext)).isTrue();
|
||||
assertThat(this.observationConvention.supportsContext(new Observation.Context())).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldHaveLowCardinalityKeyValues() {
|
||||
ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments("input")
|
||||
.build();
|
||||
assertThat(this.observationConvention.getLowCardinalityKeyValues(observationContext)).contains(
|
||||
KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.TOOL_DEFINITION_NAME.asString(),
|
||||
"toolA"),
|
||||
KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_OPERATION_TYPE.asString(),
|
||||
AiOperationType.FRAMEWORK.value()),
|
||||
KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.AI_PROVIDER.asString(),
|
||||
AiProvider.SPRING_AI.value()),
|
||||
KeyValue.of(ToolCallingObservationDocumentation.LowCardinalityKeyNames.SPRING_AI_KIND,
|
||||
SpringAiKind.TOOL_CALL.value()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldHaveHighCardinalityKeyValues() {
|
||||
String toolCallInput = """
|
||||
{
|
||||
"lizard": "George"
|
||||
}
|
||||
""";
|
||||
ToolCallingObservationContext observationContext = ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments(toolCallInput)
|
||||
.toolCallResult("Mission accomplished!")
|
||||
.build();
|
||||
assertThat(this.observationConvention.getHighCardinalityKeyValues(observationContext)).contains(
|
||||
KeyValue.of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_DESCRIPTION
|
||||
.asString(), "description"),
|
||||
KeyValue.of(
|
||||
ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_DEFINITION_SCHEMA.asString(),
|
||||
"{}"));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import io.micrometer.common.KeyValue;
|
||||
import io.micrometer.observation.Observation;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.tool.definition.ToolDefinition;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link ToolCallingContentObservationFilter}.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
*/
|
||||
class ToolCallingContentObservationFilterTests {
|
||||
|
||||
ToolCallingContentObservationFilter observationFilter = new ToolCallingContentObservationFilter();
|
||||
|
||||
@Test
|
||||
void whenNotSupportedObservationContextThenReturnOriginalContext() {
|
||||
var expectedContext = new Observation.Context();
|
||||
var actualContext = this.observationFilter.map(expectedContext);
|
||||
|
||||
assertThat(actualContext).isEqualTo(expectedContext);
|
||||
}
|
||||
|
||||
@Test
|
||||
void augmentContext() {
|
||||
var originalContext = ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments("input")
|
||||
.toolCallResult("result")
|
||||
.build();
|
||||
var augmentedContext = this.observationFilter.map(originalContext);
|
||||
|
||||
assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue
|
||||
.of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS.asString(), "input"));
|
||||
assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue
|
||||
.of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT.asString(), "result"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void augmentContextWhenNullResult() {
|
||||
var originalContext = ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments("input")
|
||||
.toolCallResult("result")
|
||||
.build();
|
||||
var augmentedContext = this.observationFilter.map(originalContext);
|
||||
|
||||
assertThat(augmentedContext.getHighCardinalityKeyValues()).contains(KeyValue
|
||||
.of(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_ARGUMENTS.asString(), "input"));
|
||||
assertThat(augmentedContext.getHighCardinalityKeyValues()
|
||||
.stream()
|
||||
.filter(kv -> kv.getKey()
|
||||
.equals(ToolCallingObservationDocumentation.HighCardinalityKeyNames.TOOL_CALL_RESULT.name())))
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright 2023-2025 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.tool.observation;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.tool.definition.ToolDefinition;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link ToolCallingObservationContext}.
|
||||
*
|
||||
* @author Thomas Vitale
|
||||
*/
|
||||
class ToolCallingObservationContextTests {
|
||||
|
||||
@Test
|
||||
void whenMandatoryRequestOptionsThenReturn() {
|
||||
var observationContext = ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments("lizard")
|
||||
.build();
|
||||
assertThat(observationContext).isNotNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenToolDefinitionIsNullThenThrow() {
|
||||
assertThatThrownBy(() -> ToolCallingObservationContext.builder().toolCallArguments("lizard").build())
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("toolDefinition cannot be null");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenToolMetadataIsNullThenThrow() {
|
||||
assertThatThrownBy(() -> ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments("lizard")
|
||||
.toolMetadata(null)
|
||||
.build()).isInstanceOf(IllegalArgumentException.class).hasMessageContaining("toolMetadata cannot be null");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenToolCallInputIsNullThenThrow() {
|
||||
assertThatThrownBy(() -> ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments(null)
|
||||
.build()).isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("toolCallArguments cannot be null or empty");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenToolCallInputIsEmptyThenThrow() {
|
||||
assertThatThrownBy(() -> ToolCallingObservationContext.builder()
|
||||
.toolDefinition(ToolDefinition.builder().name("toolA").description("description").inputSchema("{}").build())
|
||||
.toolCallArguments("")
|
||||
.build()).isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("toolCallArguments cannot be null or empty");
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user