Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add proxy tool calls option to chat models

Browse Source 
This commit introduces a new proxyToolCalls option for various chat
 models in the Spring AI project. When enabled, it allows the client to
 handle function calls externally instead of being processed internally
 by Spring AI.

 The change affects multiple chat model implementations, including:
  AnthropicChatModel
  AzureOpenAiChatModel
  MiniMaxChatModel
  MistralAiChatModel
  MoonshotChatModel
  OllamaChatModel
  OpenAiChatModel
  VertexAiGeminiChatModel
  ZhiPuAiChatModel

 The proxyToolCalls option is added to the respective chat options
 classes and integrated into the AbstractToolCallSupport class for
 consistent handling across different implementations.

 The proxyToolCalls option can be set either programmatically via
 the <ModelName>ChatOptions.builder().withProxyToolCalls() method
 or the spring.ai.<model-name>.chat.options.proxy-tool-calls
 application property.

 Documentation for the new option is also updated in the relevant
 Antora pages.

 Added ITs for proxy tool calls

 Remove ChatClientPromptRequestSpec and all ChatClient.prompt() overloads can how take advantage of the full fluent API.  Docs updated

 Resolves #1367
This commit is contained in:
Christian Tzolov
2024-09-23 16:08:33 +02:00
committed by Mark Pollack
parent acb31e7b75
commit 501774925c
42 changed files with 1023 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java
View File

@@ -225,7 +225,8 @@ public class AnthropicChatModel extends AbstractToolCallSupport implements ChatM
return chatResponse;
});
if (response != null && this.isToolCall(response, Set.of("tool_use"))) {
if (!isProxyToolCalls(prompt, this.defaultOptions) && response != null
&& this.isToolCall(response, Set.of("tool_use"))) {
var toolCallConversation = handleToolCalls(prompt, response);
return this.call(new Prompt(toolCallConversation, prompt.getOptions()));
}
@@ -256,7 +257,7 @@ public class AnthropicChatModel extends AbstractToolCallSupport implements ChatM
Flux<ChatResponse> chatResponseFlux = response.switchMap(chatCompletionResponse -> {
ChatResponse chatResponse = toChatResponse(chatCompletionResponse);
if (this.isToolCall(chatResponse, Set.of("tool_use"))) {
if (!isProxyToolCalls(prompt, this.defaultOptions) && this.isToolCall(chatResponse, Set.of("tool_use"))) {
var toolCallConversation = handleToolCalls(prompt, chatResponse);
return this.stream(new Prompt(toolCallConversation, prompt.getOptions()));
}

18
models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatOptions.java
View File

@@ -77,6 +77,9 @@ public class AnthropicChatOptions implements ChatOptions, FunctionCallingOptions
@NestedConfigurationProperty
@JsonIgnore
private Set<String> functions = new HashSet<>();
@JsonIgnore
private Boolean proxyToolCalls;
// @formatter:on
public static Builder builder() {
@@ -144,6 +147,11 @@ public class AnthropicChatOptions implements ChatOptions, FunctionCallingOptions
return this;
}
public Builder withProxyToolCalls(Boolean proxyToolCalls) {
this.options.proxyToolCalls = proxyToolCalls;
return this;
}
public AnthropicChatOptions build() {
return this.options;
}
@@ -246,6 +254,15 @@ public class AnthropicChatOptions implements ChatOptions, FunctionCallingOptions
return null;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
@Override
public AnthropicChatOptions copy() {
return fromOptions(this);
@@ -261,6 +278,7 @@ public class AnthropicChatOptions implements ChatOptions, FunctionCallingOptions
.withTopK(fromOptions.getTopK())
.withFunctionCallbacks(fromOptions.getFunctionCallbacks())
.withFunctions(fromOptions.getFunctions())
.withProxyToolCalls(fromOptions.getProxyToolCalls())
.build();
}

6
models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatModel.java
View File

@@ -151,7 +151,8 @@ public class AzureOpenAiChatModel extends AbstractToolCallSupport implements Cha
ChatResponse chatResponse = toChatResponse(chatCompletions);
if (isToolCall(chatResponse, Set.of(String.valueOf(CompletionsFinishReason.TOOL_CALLS).toLowerCase()))) {
if (!isProxyToolCalls(prompt, this.defaultOptions)
&& isToolCall(chatResponse, Set.of(String.valueOf(CompletionsFinishReason.TOOL_CALLS).toLowerCase()))) {
var toolCallConversation = handleToolCalls(prompt, chatResponse);
// Recursively call the call method with the tool call message
// conversation that contains the call responses.
@@ -199,7 +200,8 @@ public class AzureOpenAiChatModel extends AbstractToolCallSupport implements Cha
ChatResponse chatResponse = toChatResponse(chatCompletions);
if (isToolCall(chatResponse, Set.of(String.valueOf(CompletionsFinishReason.TOOL_CALLS).toLowerCase()))) {
if (!isProxyToolCalls(prompt, this.defaultOptions) && isToolCall(chatResponse,
Set.of(String.valueOf(CompletionsFinishReason.TOOL_CALLS).toLowerCase()))) {
var toolCallConversation = handleToolCalls(prompt, chatResponse);
// Recursively call the call method with the tool call message
// conversation that contains the call responses.

19
models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatOptions.java
View File

@@ -31,6 +31,7 @@ import org.springframework.ai.model.function.FunctionCallback;
import org.springframework.ai.model.function.FunctionCallingOptions;
import org.springframework.boot.context.properties.NestedConfigurationProperty;
import org.springframework.util.Assert;
import org.stringtemplate.v4.compiler.CodeGenerator.primary_return;
/**
* The configuration information for a chat completions request. Completions support a
@@ -161,6 +162,9 @@ public class AzureOpenAiChatOptions implements FunctionCallingOptions, ChatOptio
@JsonIgnore
private Set<String> functions = new HashSet<>();
@JsonIgnore
private Boolean proxyToolCalls;
public static Builder builder() {
return new Builder();
}
@@ -250,6 +254,11 @@ public class AzureOpenAiChatOptions implements FunctionCallingOptions, ChatOptio
return this;
}
public Builder withProxyToolCalls(Boolean proxyToolCalls) {
this.options.proxyToolCalls = proxyToolCalls;
return this;
}
public AzureOpenAiChatOptions build() {
return this.options;
}
@@ -395,6 +404,15 @@ public class AzureOpenAiChatOptions implements FunctionCallingOptions, ChatOptio
return null;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
@Override
public AzureOpenAiChatOptions copy() {
return fromOptions(this);
@@ -413,6 +431,7 @@ public class AzureOpenAiChatOptions implements FunctionCallingOptions, ChatOptio
.withUser(fromOptions.getUser())
.withFunctionCallbacks(fromOptions.getFunctionCallbacks())
.withFunctions(fromOptions.getFunctions())
.withResponseFormat(fromOptions.getResponseFormat())
.build();
}

4
models/spring-ai-minimax/src/main/java/org/springframework/ai/minimax/MiniMaxChatModel.java
View File

@@ -190,7 +190,7 @@ public class MiniMaxChatModel extends AbstractToolCallSupport implements ChatMod
ChatResponse chatResponse = new ChatResponse(generations, from(completionEntity.getBody()));
if (isToolCall(chatResponse,
if (!isProxyToolCalls(prompt, this.defaultOptions) && isToolCall(chatResponse,
Set.of(ChatCompletionFinishReason.TOOL_CALLS.name(), ChatCompletionFinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, chatResponse);
// Recursively call the call method with the tool call message
@@ -254,7 +254,7 @@ public class MiniMaxChatModel extends AbstractToolCallSupport implements ChatMod
return chatResponse.flatMap(response -> {
if (isToolCall(response,
if (!isProxyToolCalls(prompt, this.defaultOptions) && isToolCall(response,
Set.of(ChatCompletionFinishReason.TOOL_CALLS.name(), ChatCompletionFinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, response);
// Recursively call the stream method with the tool call message

25
models/spring-ai-minimax/src/main/java/org/springframework/ai/minimax/MiniMaxChatOptions.java
View File

@@ -142,6 +142,9 @@ public class MiniMaxChatOptions implements FunctionCallingOptions, ChatOptions {
@NestedConfigurationProperty
@JsonIgnore
private Set<String> functions = new HashSet<>();
@JsonIgnore
private Boolean proxyToolCalls;
// @formatter:on
public static Builder builder() {
@@ -242,6 +245,11 @@ public class MiniMaxChatOptions implements FunctionCallingOptions, ChatOptions {
return this;
}
public Builder withProxyToolCalls(Boolean proxyToolCalls) {
this.options.proxyToolCalls = proxyToolCalls;
return this;
}
public MiniMaxChatOptions build() {
return this.options;
}
@@ -394,6 +402,15 @@ public class MiniMaxChatOptions implements FunctionCallingOptions, ChatOptions {
return null;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
@Override
public int hashCode() {
final int prime = 31;
@@ -411,6 +428,7 @@ public class MiniMaxChatOptions implements FunctionCallingOptions, ChatOptions {
result = prime * result + ((maskSensitiveInfo == null) ? 0 : maskSensitiveInfo.hashCode());
result = prime * result + ((tools == null) ? 0 : tools.hashCode());
result = prime * result + ((toolChoice == null) ? 0 : toolChoice.hashCode());
result = prime * result + ((proxyToolCalls == null) ? 0 : proxyToolCalls.hashCode());
return result;
}
@@ -501,6 +519,12 @@ public class MiniMaxChatOptions implements FunctionCallingOptions, ChatOptions {
}
else if (!toolChoice.equals(other.toolChoice))
return false;
if (this.proxyToolCalls == null) {
if (other.proxyToolCalls != null)
return false;
}
else if (!proxyToolCalls.equals(other.proxyToolCalls))
return false;
return true;
}
@@ -525,6 +549,7 @@ public class MiniMaxChatOptions implements FunctionCallingOptions, ChatOptions {
.withToolChoice(fromOptions.getToolChoice())
.withFunctionCallbacks(fromOptions.getFunctionCallbacks())
.withFunctions(fromOptions.getFunctions())
.withProxyToolCalls(fromOptions.getProxyToolCalls())
.build();
}

7
models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java
View File

@@ -183,8 +183,9 @@ public class MistralAiChatModel extends AbstractToolCallSupport implements ChatM
return chatResponse;
});
if (response != null && isToolCall(response, Set.of(MistralAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
MistralAiApi.ChatCompletionFinishReason.STOP.name()))) {
if (!isProxyToolCalls(prompt, this.defaultOptions) && response != null
&& isToolCall(response, Set.of(MistralAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
MistralAiApi.ChatCompletionFinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, response);
// Recursively call the call method with the tool call message
// conversation that contains the call responses.
@@ -255,7 +256,7 @@ public class MistralAiChatModel extends AbstractToolCallSupport implements ChatM
// @formatter:off
Flux<ChatResponse> chatResponseFlux = chatResponse.flatMap(response -> {
if (isToolCall(response, Set.of(MistralAiApi.ChatCompletionFinishReason.TOOL_CALLS.name()))) {
if (!isProxyToolCalls(prompt, this.defaultOptions) && isToolCall(response, Set.of(MistralAiApi.ChatCompletionFinishReason.TOOL_CALLS.name()))) {
var toolCallConversation = handleToolCalls(prompt, response);
// Recursively call the stream method with the tool call message
// conversation that contains the call responses.

124
models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatOptions.java
View File

@@ -135,6 +135,9 @@ public class MistralAiChatOptions implements FunctionCallingOptions, ChatOptions
@JsonIgnore
private Set<String> functions = new HashSet<>();
@JsonIgnore
private Boolean proxyToolCalls;
public static Builder builder() {
return new Builder();
}
@@ -215,6 +218,11 @@ public class MistralAiChatOptions implements FunctionCallingOptions, ChatOptions
return this;
}
public Builder withProxyToolCalls(Boolean proxyToolCalls) {
this.options.proxyToolCalls = proxyToolCalls;
return this;
}
public MistralAiChatOptions build() {
return this.options;
}
@@ -356,6 +364,15 @@ public class MistralAiChatOptions implements FunctionCallingOptions, ChatOptions
return null;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
@Override
public MistralAiChatOptions copy() {
return fromOptions(this);
@@ -374,7 +391,114 @@ public class MistralAiChatOptions implements FunctionCallingOptions, ChatOptions
.withToolChoice(fromOptions.getToolChoice())
.withFunctionCallbacks(fromOptions.getFunctionCallbacks())
.withFunctions(fromOptions.getFunctions())
.withProxyToolCalls(fromOptions.getProxyToolCalls())
.build();
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((model == null) ? 0 : model.hashCode());
result = prime * result + ((temperature == null) ? 0 : temperature.hashCode());
result = prime * result + ((topP == null) ? 0 : topP.hashCode());
result = prime * result + ((maxTokens == null) ? 0 : maxTokens.hashCode());
result = prime * result + ((safePrompt == null) ? 0 : safePrompt.hashCode());
result = prime * result + ((randomSeed == null) ? 0 : randomSeed.hashCode());
result = prime * result + ((responseFormat == null) ? 0 : responseFormat.hashCode());
result = prime * result + ((stop == null) ? 0 : stop.hashCode());
result = prime * result + ((tools == null) ? 0 : tools.hashCode());
result = prime * result + ((toolChoice == null) ? 0 : toolChoice.hashCode());
result = prime * result + ((functionCallbacks == null) ? 0 : functionCallbacks.hashCode());
result = prime * result + ((functions == null) ? 0 : functions.hashCode());
result = prime * result + ((proxyToolCalls == null) ? 0 : proxyToolCalls.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
MistralAiChatOptions other = (MistralAiChatOptions) obj;
if (model == null) {
if (other.model != null)
return false;
}
else if (!model.equals(other.model))
return false;
if (temperature == null) {
if (other.temperature != null)
return false;
}
else if (!temperature.equals(other.temperature))
return false;
if (topP == null) {
if (other.topP != null)
return false;
}
else if (!topP.equals(other.topP))
return false;
if (maxTokens == null) {
if (other.maxTokens != null)
return false;
}
else if (!maxTokens.equals(other.maxTokens))
return false;
if (safePrompt == null) {
if (other.safePrompt != null)
return false;
}
else if (!safePrompt.equals(other.safePrompt))
return false;
if (randomSeed == null) {
if (other.randomSeed != null)
return false;
}
else if (!randomSeed.equals(other.randomSeed))
return false;
if (responseFormat == null) {
if (other.responseFormat != null)
return false;
}
else if (!responseFormat.equals(other.responseFormat))
return false;
if (stop == null) {
if (other.stop != null)
return false;
}
else if (!stop.equals(other.stop))
return false;
if (tools == null) {
if (other.tools != null)
return false;
}
else if (!tools.equals(other.tools))
return false;
if (toolChoice != other.toolChoice)
return false;
if (functionCallbacks == null) {
if (other.functionCallbacks != null)
return false;
}
else if (!functionCallbacks.equals(other.functionCallbacks))
return false;
if (functions == null) {
if (other.functions != null)
return false;
}
else if (!functions.equals(other.functions))
return false;
if (proxyToolCalls == null) {
if (other.proxyToolCalls != null)
return false;
}
else if (!proxyToolCalls.equals(other.proxyToolCalls))
return false;
return true;
}
}

8
models/spring-ai-moonshot/src/main/java/org/springframework/ai/moonshot/MoonshotChatModel.java
View File

@@ -164,8 +164,9 @@ public class MoonshotChatModel extends AbstractToolCallSupport implements ChatMo
ChatResponse chatResponse = new ChatResponse(generations, from(completionEntity.getBody()));
if (isToolCall(chatResponse, Set.of(MoonshotApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
MoonshotApi.ChatCompletionFinishReason.STOP.name()))) {
if (!isProxyToolCalls(prompt, this.defaultOptions)
&& isToolCall(chatResponse, Set.of(MoonshotApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
MoonshotApi.ChatCompletionFinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, chatResponse);
// Recursively call the call method with the tool call message
// conversation that contains the call responses.
@@ -228,7 +229,8 @@ public class MoonshotChatModel extends AbstractToolCallSupport implements ChatMo
return chatResponse.flatMap(response -> {
if (isToolCall(response, Set.of(ChatCompletionFinishReason.TOOL_CALLS.name(), "stop"))) {
if (!isProxyToolCalls(prompt, this.defaultOptions)
&& isToolCall(response, Set.of(ChatCompletionFinishReason.TOOL_CALLS.name(), "stop"))) {
var toolCallConversation = handleToolCalls(prompt, response);
// Recursively call the stream method with the tool call message
// conversation that contains the call responses.

24
models/spring-ai-moonshot/src/main/java/org/springframework/ai/moonshot/MoonshotChatOptions.java
View File

@@ -137,6 +137,9 @@ public class MoonshotChatOptions implements FunctionCallingOptions, ChatOptions
*/
private @JsonProperty("user") String user;
@JsonIgnore
private Boolean proxyToolCalls;
@Override
public List<FunctionCallback> getFunctionCallbacks() {
return this.functionCallbacks;
@@ -244,6 +247,11 @@ public class MoonshotChatOptions implements FunctionCallingOptions, ChatOptions
return this;
}
public Builder withProxyToolCalls(Boolean proxyToolCalls) {
this.options.proxyToolCalls = proxyToolCalls;
return this;
}
public MoonshotChatOptions build() {
return this.options;
}
@@ -345,6 +353,15 @@ public class MoonshotChatOptions implements FunctionCallingOptions, ChatOptions
return null;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
@Override
public MoonshotChatOptions copy() {
return builder().withModel(this.model)
@@ -360,6 +377,7 @@ public class MoonshotChatOptions implements FunctionCallingOptions, ChatOptions
.withToolChoice(this.toolChoice)
.withFunctionCallbacks(this.functionCallbacks)
.withFunctions(this.functions)
.withProxyToolCalls(this.proxyToolCalls)
.build();
}
@@ -376,6 +394,7 @@ public class MoonshotChatOptions implements FunctionCallingOptions, ChatOptions
result = prime * result + ((temperature == null) ? 0 : temperature.hashCode());
result = prime * result + ((topP == null) ? 0 : topP.hashCode());
result = prime * result + ((user == null) ? 0 : user.hashCode());
result = prime * result + ((proxyToolCalls == null) ? 0 : proxyToolCalls.hashCode());
return result;
}
@@ -441,6 +460,11 @@ public class MoonshotChatOptions implements FunctionCallingOptions, ChatOptions
}
else if (!this.user.equals(other.user))
return false;
if (this.proxyToolCalls == null) {
return other.proxyToolCalls == null;
}
else if (!this.proxyToolCalls.equals(other.proxyToolCalls))
return false;
return true;
}

3
models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java
View File

@@ -162,7 +162,8 @@ public class OllamaChatModel extends AbstractToolCallSupport implements ChatMode
});
if (response != null && isToolCall(response, Set.of("stop"))) {
if (!isProxyToolCalls(prompt, this.defaultOptions) && response != null
&& isToolCall(response, Set.of("stop"))) {
var toolCallConversation = handleToolCalls(prompt, response);
// Recursively call the call method with the tool call message
// conversation that contains the call responses.

23
models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java
View File

@@ -270,7 +270,7 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed
/**
* Truncates the end of each input to fit within context length. Returns error if false and context length is exceeded.
* Truncates the end of each input to fit within context length. Returns error if false and context length is exceeded.
* Defaults to true.
*/
@JsonProperty("truncate") private Boolean truncate;
@@ -297,6 +297,8 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed
@JsonIgnore
private Set<String> functions = new HashSet<>();
@JsonIgnore
private Boolean proxyToolCalls;
public static OllamaOptions builder() {
return new OllamaOptions();
@@ -495,6 +497,11 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed
return this;
}
public OllamaOptions withProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
return this;
}
// -------------------
// Getters and Setters
// -------------------
@@ -816,6 +823,15 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed
return null;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
/**
* Convert the {@link OllamaOptions} object to a {@link Map} of key/value pairs.
* @return The {@link Map} of key/value pairs.
@@ -884,6 +900,7 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed
.withPenalizeNewline(fromOptions.getPenalizeNewline())
.withStop(fromOptions.getStop())
.withFunctions(fromOptions.getFunctions())
.withProxyToolCalls(fromOptions.getProxyToolCalls())
.withFunctionCallbacks(fromOptions.getFunctionCallbacks());
}
// @formatter:on
@@ -913,7 +930,7 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed
&& Objects.equals(mirostatTau, that.mirostatTau) && Objects.equals(mirostatEta, that.mirostatEta)
&& Objects.equals(penalizeNewline, that.penalizeNewline) && Objects.equals(stop, that.stop)
&& Objects.equals(functionCallbacks, that.functionCallbacks)
&& Objects.equals(functions, that.functions);
&& Objects.equals(proxyToolCalls, that.proxyToolCalls) && Objects.equals(functions, that.functions);
}
@Override
@@ -923,7 +940,7 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed
this.useMMap, this.useMLock, this.numThread, this.numKeep, this.seed, this.numPredict, this.topK,
this.topP, tfsZ, this.typicalP, this.repeatLastN, this.temperature, this.repeatPenalty,
this.presencePenalty, this.frequencyPenalty, this.mirostat, this.mirostatTau, this.mirostatEta,
this.penalizeNewline, this.stop, this.functionCallbacks, this.functions);
this.penalizeNewline, this.stop, this.functionCallbacks, this.functions, this.proxyToolCalls);
}
}

35
models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java
View File

@@ -15,9 +15,16 @@
*/
package org.springframework.ai.openai;
import io.micrometer.observation.Observation;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.contextpropagation.ObservationThreadLocalAccessor;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.messages.AssistantMessage;
@@ -63,19 +70,13 @@ import org.springframework.util.CollectionUtils;
import org.springframework.util.MimeType;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StringUtils;
import io.micrometer.observation.Observation;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.contextpropagation.ObservationThreadLocalAccessor;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* {@link ChatModel} and {@link StreamingChatModel} implementation for {@literal OpenAI}
* backed by {@link OpenAiApi}.
@@ -189,6 +190,7 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
public OpenAiChatModel(OpenAiApi openAiApi, OpenAiChatOptions options,
FunctionCallbackContext functionCallbackContext, List<FunctionCallback> toolFunctionCallbacks,
RetryTemplate retryTemplate, ObservationRegistry observationRegistry) {
super(functionCallbackContext, options, toolFunctionCallbacks);
Assert.notNull(openAiApi, "OpenAiApi must not be null");
@@ -259,8 +261,9 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
});
if (response != null && isToolCall(response, Set.of(OpenAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
OpenAiApi.ChatCompletionFinishReason.STOP.name()))) {
if (!isProxyToolCalls(prompt, this.defaultOptions)
&& isToolCall(response, Set.of(OpenAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
OpenAiApi.ChatCompletionFinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, response);
// Recursively call the call method with the tool call message
// conversation that contains the call responses.
@@ -330,7 +333,7 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
// @formatter:off
Flux<ChatResponse> flux = chatResponse.flatMap(response -> {
if (isToolCall(response, Set.of(OpenAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
if (!isProxyToolCalls(prompt, this.defaultOptions) && isToolCall(response, Set.of(OpenAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
OpenAiApi.ChatCompletionFinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, response);
// Recursively call the stream method with the tool call message

203
models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java
View File

@@ -20,6 +20,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.springframework.ai.chat.prompt.ChatOptions;
@@ -171,6 +172,14 @@ public class OpenAiChatOptions implements FunctionCallingOptions, ChatOptions {
@JsonIgnore
private Set<String> functions = new HashSet<>();
/**
* If true, the Spring AI will not handle the function calls internally, but will proxy them to the client.
* It is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results.
* If false, the Spring AI will handle the function calls internally.
*/
@JsonIgnore
private Boolean proxyToolCalls;
/**
* Optional HTTP headers to be added to the chat completion request.
*/
@@ -307,8 +316,12 @@ public class OpenAiChatOptions implements FunctionCallingOptions, ChatOptions {
return this;
}
public Builder withProxyToolCalls(Boolean proxyToolCalls) {
this.options.proxyToolCalls = proxyToolCalls;
return this;
}
public Builder withHttpHeaders(Map<String, String> httpHeaders) {
Assert.notNull(httpHeaders, "HTTP headers must not be null");
this.options.httpHeaders = httpHeaders;
return this;
}
@@ -468,6 +481,15 @@ public class OpenAiChatOptions implements FunctionCallingOptions, ChatOptions {
return this.toolChoice;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
public void setToolChoice(String toolChoice) {
this.toolChoice = toolChoice;
}
@@ -521,152 +543,6 @@ public class OpenAiChatOptions implements FunctionCallingOptions, ChatOptions {
return null;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((model == null) ? 0 : model.hashCode());
result = prime * result + ((frequencyPenalty == null) ? 0 : frequencyPenalty.hashCode());
result = prime * result + ((logitBias == null) ? 0 : logitBias.hashCode());
result = prime * result + ((logprobs == null) ? 0 : logprobs.hashCode());
result = prime * result + ((topLogprobs == null) ? 0 : topLogprobs.hashCode());
result = prime * result + ((maxTokens == null) ? 0 : maxTokens.hashCode());
result = prime * result + ((n == null) ? 0 : n.hashCode());
result = prime * result + ((presencePenalty == null) ? 0 : presencePenalty.hashCode());
result = prime * result + ((responseFormat == null) ? 0 : responseFormat.hashCode());
result = prime * result + ((streamOptions == null) ? 0 : streamOptions.hashCode());
result = prime * result + ((seed == null) ? 0 : seed.hashCode());
result = prime * result + ((stop == null) ? 0 : stop.hashCode());
result = prime * result + ((temperature == null) ? 0 : temperature.hashCode());
result = prime * result + ((topP == null) ? 0 : topP.hashCode());
result = prime * result + ((tools == null) ? 0 : tools.hashCode());
result = prime * result + ((toolChoice == null) ? 0 : toolChoice.hashCode());
result = prime * result + ((user == null) ? 0 : user.hashCode());
result = prime * result + ((parallelToolCalls == null) ? 0 : parallelToolCalls.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
OpenAiChatOptions other = (OpenAiChatOptions) obj;
if (this.model == null) {
if (other.model != null)
return false;
}
else if (!model.equals(other.model))
return false;
if (this.frequencyPenalty == null) {
if (other.frequencyPenalty != null)
return false;
}
else if (!this.frequencyPenalty.equals(other.frequencyPenalty))
return false;
if (this.logitBias == null) {
if (other.logitBias != null)
return false;
}
else if (!this.logitBias.equals(other.logitBias))
return false;
if (this.logprobs == null) {
if (other.logprobs != null)
return false;
}
else if (!this.logprobs.equals(other.logprobs))
return false;
if (this.topLogprobs == null) {
if (other.topLogprobs != null)
return false;
}
else if (!this.topLogprobs.equals(other.topLogprobs))
return false;
if (this.maxTokens == null) {
if (other.maxTokens != null)
return false;
}
else if (!this.maxTokens.equals(other.maxTokens))
return false;
if (this.n == null) {
if (other.n != null)
return false;
}
else if (!this.n.equals(other.n))
return false;
if (this.presencePenalty == null) {
if (other.presencePenalty != null)
return false;
}
else if (!this.presencePenalty.equals(other.presencePenalty))
return false;
if (this.responseFormat == null) {
if (other.responseFormat != null)
return false;
}
else if (!this.responseFormat.equals(other.responseFormat))
return false;
if (this.streamOptions == null) {
if (other.streamOptions != null)
return false;
}
else if (!this.streamOptions.equals(other.streamOptions))
return false;
if (this.seed == null) {
if (other.seed != null)
return false;
}
else if (!this.seed.equals(other.seed))
return false;
if (this.stop == null) {
if (other.stop != null)
return false;
}
else if (!stop.equals(other.stop))
return false;
if (this.temperature == null) {
if (other.temperature != null)
return false;
}
else if (!this.temperature.equals(other.temperature))
return false;
if (this.topP == null) {
if (other.topP != null)
return false;
}
else if (!topP.equals(other.topP))
return false;
if (this.tools == null) {
if (other.tools != null)
return false;
}
else if (!tools.equals(other.tools))
return false;
if (this.toolChoice == null) {
if (other.toolChoice != null)
return false;
}
else if (!toolChoice.equals(other.toolChoice))
return false;
if (this.user == null) {
if (other.user != null)
return false;
}
else if (!this.user.equals(other.user))
return false;
else if (this.parallelToolCalls == null) {
if (other.parallelToolCalls != null)
return false;
}
else if (!this.parallelToolCalls.equals(other.parallelToolCalls))
return false;
return true;
}
@Override
public OpenAiChatOptions copy() {
return OpenAiChatOptions.fromOptions(this);
@@ -695,9 +571,42 @@ public class OpenAiChatOptions implements FunctionCallingOptions, ChatOptions {
.withFunctionCallbacks(fromOptions.getFunctionCallbacks())
.withFunctions(fromOptions.getFunctions())
.withHttpHeaders(fromOptions.getHttpHeaders())
.withProxyToolCalls(fromOptions.getProxyToolCalls())
.build();
}
@Override
public int hashCode() {
return Objects.hash(this.model, this.frequencyPenalty, this.logitBias, this.logprobs, this.topLogprobs,
this.maxTokens, this.n, this.presencePenalty, this.responseFormat, this.streamOptions, this.seed,
this.stop, this.temperature, this.topP, this.tools, this.toolChoice, this.user, this.parallelToolCalls,
this.functionCallbacks, this.functions, this.httpHeaders, this.proxyToolCalls);
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
OpenAiChatOptions other = (OpenAiChatOptions) o;
return Objects.equals(this.model, other.model) && Objects.equals(this.frequencyPenalty, other.frequencyPenalty)
&& Objects.equals(this.logitBias, other.logitBias) && Objects.equals(this.logprobs, other.logprobs)
&& Objects.equals(this.topLogprobs, other.topLogprobs)
&& Objects.equals(this.maxTokens, other.maxTokens) && Objects.equals(this.n, other.n)
&& Objects.equals(this.presencePenalty, other.presencePenalty)
&& Objects.equals(this.responseFormat, other.responseFormat)
&& Objects.equals(this.streamOptions, other.streamOptions) && Objects.equals(this.seed, other.seed)
&& Objects.equals(this.stop, other.stop) && Objects.equals(this.temperature, other.temperature)
&& Objects.equals(this.topP, other.topP) && Objects.equals(this.tools, other.tools)
&& Objects.equals(this.toolChoice, other.toolChoice) && Objects.equals(this.user, other.user)
&& Objects.equals(this.parallelToolCalls, other.parallelToolCalls)
&& Objects.equals(this.functionCallbacks, other.functionCallbacks)
&& Objects.equals(this.functions, other.functions)
&& Objects.equals(this.httpHeaders, other.httpHeaders)
&& Objects.equals(this.proxyToolCalls, other.proxyToolCalls);
}
@Override
public String toString() {
return "OpenAiChatOptions: " + ModelOptionsUtils.toJsonString(this);

7
models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java
View File

@@ -756,6 +756,8 @@ public class OpenAiApi {
/**
* The relevant tool call.
*
* @param index The index of the tool call in the list of tool calls. Required in
* case of streaming.
* @param id The ID of the tool call. This ID must be referenced when you submit
* the tool outputs in using the Submit tool outputs to run endpoint.
* @param type The type of tool call the output is required for. For now, this is
@@ -764,9 +766,14 @@ public class OpenAiApi {
*/
@JsonInclude(Include.NON_NULL)
public record ToolCall(// @formatter:off
@JsonProperty("index") Integer index,
@JsonProperty("id") String id,
@JsonProperty("type") String type,
@JsonProperty("function") ChatCompletionFunction function) {// @formatter:on
public ToolCall(String id, String type, ChatCompletionFunction function) {
this(null, id, type, function);
}
}
/**

2
models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java
View File

@@ -64,7 +64,7 @@ import reactor.core.publisher.Flux;
@SpringBootTest(classes = OpenAiTestConfiguration.class)
@EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+")
class OpenAiChatModelIT extends AbstractIT {
public class OpenAiChatModelIT extends AbstractIT {
private static final Logger logger = LoggerFactory.getLogger(OpenAiChatModelIT.class);

372
models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelProxyToolCallsIT.java Normal file
View File

@@ -0,0 +1,372 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.openai.chat;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.ToolResponseMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.model.ModelOptionsUtils;
import org.springframework.ai.model.function.ToolCallHelper;
import org.springframework.ai.model.function.FunctionCallback;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.ai.retry.RetryUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.util.CollectionUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.observation.ObservationRegistry;
import reactor.core.publisher.Flux;
@SpringBootTest(classes = OpenAiChatModelProxyToolCallsIT.Config.class)
@EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+")
class OpenAiChatModelProxyToolCallsIT {
private static final Logger logger = LoggerFactory.getLogger(OpenAiChatModelIT.class);
private static final String DEFAULT_MODEL = "gpt-4o-mini";
@Autowired
private OpenAiChatModel chatModel;
// Helper class that reuses some of the {@link AbstractToolCallSupport} functionality
// to help to implement the function call handling logic on the client side.
private ToolCallHelper toolCallHelper = new ToolCallHelper();
// Function which will be called by the AI model.
private String getWeatherInLocation(String location, String unit) {
double temperature = 0;
if (location.contains("Paris")) {
temperature = 15;
}
else if (location.contains("Tokyo")) {
temperature = 10;
}
else if (location.contains("San Francisco")) {
temperature = 30;
}
return String.format("The weather in %s is %s%s", location, temperature, unit);
}
FunctionCallback functionDefinition = new ToolCallHelper.FunctionDefinition("getWeatherInLocation",
"Get the weather in location", """
{
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state e.g. San Francisco, CA"
},
"unit": {
"type": "string",
"enum": ["C", "F"]
}
},
"required": ["location", "unit"]
}
""");
@Test
void functionCall() throws JsonMappingException, JsonProcessingException {
List<Message> messages = List
.of(new UserMessage("What's the weather like in San Francisco, Tokyo, and Paris?"));
var promptOptions = OpenAiChatOptions.builder().withFunctionCallbacks(List.of(functionDefinition)).build();
var prompt = new Prompt(messages, promptOptions);
boolean isToolCall = false;
ChatResponse chatResponse = null;
do {
chatResponse = chatModel.call(prompt);
// We will have to convert the chatResponse into OpenAI assistant message.
// Note that the tool call check could be platform specific because the finish
// reasons.
isToolCall = toolCallHelper.isToolCall(chatResponse,
Set.of(OpenAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
OpenAiApi.ChatCompletionFinishReason.STOP.name()));
if (isToolCall) {
Optional<Generation> toolCallGeneration = chatResponse.getResults()
.stream()
.filter(g -> !CollectionUtils.isEmpty(g.getOutput().getToolCalls()))
.findFirst();
assertThat(toolCallGeneration).isNotEmpty();
AssistantMessage assistantMessage = toolCallGeneration.get().getOutput();
List<ToolResponseMessage.ToolResponse> toolResponses = new ArrayList<>();
for (AssistantMessage.ToolCall toolCall : assistantMessage.getToolCalls()) {
var functionName = toolCall.name();
assertThat(functionName).isEqualTo("getWeatherInLocation");
String functionArguments = toolCall.arguments();
@SuppressWarnings("unchecked")
Map<String, String> argumentsMap = new ObjectMapper().readValue(functionArguments, Map.class);
String functionResponse = getWeatherInLocation(argumentsMap.get("location").toString(),
argumentsMap.get("unit").toString());
toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), functionName,
ModelOptionsUtils.toJsonString(functionResponse)));
}
ToolResponseMessage toolMessageResponse = new ToolResponseMessage(toolResponses, Map.of());
List<Message> toolCallConversation = toolCallHelper.buildToolCallConversation(prompt.getInstructions(),
assistantMessage, toolMessageResponse);
assertThat(toolCallConversation).isNotEmpty();
prompt = new Prompt(toolCallConversation, prompt.getOptions());
}
}
while (isToolCall);
logger.info("Response: {}", chatResponse);
assertThat(chatResponse.getResult().getOutput().getContent()).contains("30", "10", "15");
}
@Test
void functionStream() throws JsonMappingException, JsonProcessingException {
List<Message> messages = List
.of(new UserMessage("What's the weather like in San Francisco, Tokyo, and Paris?"));
var promptOptions = OpenAiChatOptions.builder().withFunctionCallbacks(List.of(functionDefinition)).build();
var prompt = new Prompt(messages, promptOptions);
String response = processToolCall(prompt, Set.of(OpenAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
OpenAiApi.ChatCompletionFinishReason.STOP.name()), toolCall -> {
var functionName = toolCall.name();
assertThat(functionName).isEqualTo("getWeatherInLocation");
String functionArguments = toolCall.arguments();
Map<String, String> argumentsMap = getFunctionArguments(functionArguments);
String functionResponse = getWeatherInLocation(argumentsMap.get("location").toString(),
argumentsMap.get("unit").toString());
return functionResponse;
})
.collectList()
.block()
.stream()
.map(cr -> cr.getResult().getOutput().getContent())
.collect(Collectors.joining());
logger.info("Response: {}", response);
assertThat(response).contains("30", "10", "15");
}
private Flux<ChatResponse> processToolCall(Prompt prompt, Set<String> finishReasons,
Function<AssistantMessage.ToolCall, String> customFunction) {
Flux<ChatResponse> chatResponses = chatModel.stream(prompt);
return chatResponses.flatMap(chatResponse -> {
boolean isToolCall = toolCallHelper.isToolCall(chatResponse, finishReasons);
if (isToolCall) {
Optional<Generation> toolCallGeneration = chatResponse.getResults()
.stream()
.filter(g -> !CollectionUtils.isEmpty(g.getOutput().getToolCalls()))
.findFirst();
assertThat(toolCallGeneration).isNotEmpty();
AssistantMessage assistantMessage = toolCallGeneration.get().getOutput();
List<ToolResponseMessage.ToolResponse> toolResponses = new ArrayList<>();
for (AssistantMessage.ToolCall toolCall : assistantMessage.getToolCalls()) {
String functionResponse = customFunction.apply(toolCall);
toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolCall.name(),
ModelOptionsUtils.toJsonString(functionResponse)));
}
ToolResponseMessage toolMessageResponse = new ToolResponseMessage(toolResponses, Map.of());
List<Message> toolCallConversation = toolCallHelper.buildToolCallConversation(prompt.getInstructions(),
assistantMessage, toolMessageResponse);
assertThat(toolCallConversation).isNotEmpty();
var prompt2 = new Prompt(toolCallConversation, prompt.getOptions());
return processToolCall(prompt2, finishReasons, customFunction);
}
return Flux.just(chatResponse);
});
}
@Test
void functionCall2() throws JsonMappingException, JsonProcessingException {
List<Message> messages = List
.of(new UserMessage("What's the weather like in San Francisco, Tokyo, and Paris?"));
var promptOptions = OpenAiChatOptions.builder().withFunctionCallbacks(List.of(functionDefinition)).build();
var prompt = new Prompt(messages, promptOptions);
ChatResponse chatResponse = toolCallHelper.processCall(chatModel, prompt,
Set.of(OpenAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
OpenAiApi.ChatCompletionFinishReason.STOP.name()),
toolCall -> {
var functionName = toolCall.name();
assertThat(functionName).isEqualTo("getWeatherInLocation");
String functionArguments = toolCall.arguments();
Map<String, String> argumentsMap = getFunctionArguments(functionArguments);
String functionResponse = getWeatherInLocation(argumentsMap.get("location").toString(),
argumentsMap.get("unit").toString());
return functionResponse;
});
logger.info("Response: {}", chatResponse);
assertThat(chatResponse.getResult().getOutput().getContent()).contains("30", "10", "15");
}
@Test
void functionStream2() throws JsonMappingException, JsonProcessingException {
List<Message> messages = List
.of(new UserMessage("What's the weather like in San Francisco, Tokyo, and Paris?"));
var promptOptions = OpenAiChatOptions.builder().withFunctionCallbacks(List.of(functionDefinition)).build();
var prompt = new Prompt(messages, promptOptions);
Flux<ChatResponse> responses = toolCallHelper.processStream(chatModel, prompt,
Set.of(OpenAiApi.ChatCompletionFinishReason.TOOL_CALLS.name(),
OpenAiApi.ChatCompletionFinishReason.STOP.name()),
toolCall -> {
var functionName = toolCall.name();
assertThat(functionName).isEqualTo("getWeatherInLocation");
String functionArguments = toolCall.arguments();
Map<String, String> argumentsMap = getFunctionArguments(functionArguments);
String functionResponse = getWeatherInLocation(argumentsMap.get("location").toString(),
argumentsMap.get("unit").toString());
return functionResponse;
});
String response = responses.collectList()
.block()
.stream()
.map(cr -> cr.getResult().getOutput().getContent())
.collect(Collectors.joining());
logger.info("Response: {}", response);
assertThat(response).contains("30", "10", "15");
}
@SuppressWarnings("unchecked")
private static Map<String, String> getFunctionArguments(String functionArguments) {
try {
return new ObjectMapper().readValue(functionArguments, Map.class);
}
catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
@SpringBootConfiguration
static class Config {
@Bean
public OpenAiApi chatCompletionApi() {
return new OpenAiApi(System.getenv("OPENAI_API_KEY"));
}
@Bean
public OpenAiChatModel openAiClient(OpenAiApi openAiApi, List<FunctionCallback> toolFunctionCallbacks) {
// enable the proxy tool calls option.
var options = OpenAiChatOptions.builder().withModel(DEFAULT_MODEL).withProxyToolCalls(true).build();
return new OpenAiChatModel(openAiApi, options, null, toolFunctionCallbacks,
RetryUtils.DEFAULT_RETRY_TEMPLATE, ObservationRegistry.NOOP);
}
}
}

4
models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/GroqWithOpenAiChatModelIT.java → models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/proxy/GroqWithOpenAiChatModelIT.java
View File

@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.openai.chat;
package org.springframework.ai.openai.chat.proxy;
import static org.assertj.core.api.Assertions.assertThat;
@@ -50,6 +50,8 @@ import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.ai.openai.api.tool.MockWeatherService;
import org.springframework.ai.openai.chat.ActorsFilms;
import org.springframework.ai.openai.chat.OpenAiChatModelIT;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringBootConfiguration;

4
models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OllamaWithOpenAiChatModelIT.java → models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/proxy/OllamaWithOpenAiChatModelIT.java
View File

@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.openai.chat;
package org.springframework.ai.openai.chat.proxy;
import static org.assertj.core.api.Assertions.assertThat;
@@ -50,6 +50,8 @@ import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.ai.openai.api.tool.MockWeatherService;
import org.springframework.ai.openai.chat.ActorsFilms;
import org.springframework.ai.openai.chat.OpenAiChatModelIT;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringBootConfiguration;

5
models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModel.java
View File

@@ -180,7 +180,8 @@ public class VertexAiGeminiChatModel extends AbstractToolCallSupport implements
ChatResponse chatResponse = new ChatResponse(generations, toChatResponseMetadata(response));
if (isToolCall(chatResponse, Set.of(FinishReason.STOP.name()))) {
if (!isProxyToolCalls(prompt, this.defaultOptions)
&& isToolCall(chatResponse, Set.of(FinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, chatResponse);
// Recursively call the call method with the tool call message
// conversation that contains the call responses.
@@ -209,7 +210,7 @@ public class VertexAiGeminiChatModel extends AbstractToolCallSupport implements
ChatResponse chatResponse = new ChatResponse(generations, toChatResponseMetadata(response));
if (isToolCall(chatResponse,
if (!isProxyToolCalls(prompt, this.defaultOptions) && isToolCall(chatResponse,
Set.of(FinishReason.STOP.name(), FinishReason.FINISH_REASON_UNSPECIFIED.name()))) {
var toolCallConversation = handleToolCalls(prompt, chatResponse);
// Recursively call the stream method with the tool call message

21
models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatOptions.java
View File

@@ -115,6 +115,8 @@ public class VertexAiGeminiChatOptions implements FunctionCallingOptions, ChatOp
@JsonIgnore
private boolean googleSearchRetrieval = false;
@JsonIgnore
private Boolean proxyToolCalls;
// @formatter:on
@@ -194,6 +196,11 @@ public class VertexAiGeminiChatOptions implements FunctionCallingOptions, ChatOp
return this;
}
public Builder withProxyToolCalls(boolean proxyToolCalls) {
this.options.proxyToolCalls = proxyToolCalls;
return this;
}
public VertexAiGeminiChatOptions build() {
return this.options;
}
@@ -321,6 +328,15 @@ public class VertexAiGeminiChatOptions implements FunctionCallingOptions, ChatOp
this.googleSearchRetrieval = googleSearchRetrieval;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
@Override
public boolean equals(Object o) {
if (this == o)
@@ -333,13 +349,13 @@ public class VertexAiGeminiChatOptions implements FunctionCallingOptions, ChatOp
&& Objects.equals(maxOutputTokens, that.maxOutputTokens) && Objects.equals(model, that.model)
&& Objects.equals(responseMimeType, that.responseMimeType)
&& Objects.equals(functionCallbacks, that.functionCallbacks)
&& Objects.equals(functions, that.functions);
&& Objects.equals(functions, that.functions) && Objects.equals(proxyToolCalls, that.proxyToolCalls);
}
@Override
public int hashCode() {
return Objects.hash(stopSequences, temperature, topP, topK, candidateCount, maxOutputTokens, model,
responseMimeType, functionCallbacks, functions, googleSearchRetrieval);
responseMimeType, functionCallbacks, functions, googleSearchRetrieval, proxyToolCalls);
}
@Override
@@ -370,6 +386,7 @@ public class VertexAiGeminiChatOptions implements FunctionCallingOptions, ChatOp
options.setFunctions(fromOptions.getFunctions());
options.setResponseMimeType(fromOptions.getResponseMimeType());
options.setGoogleSearchRetrieval(fromOptions.getGoogleSearchRetrieval());
options.setProxyToolCalls(fromOptions.getProxyToolCalls());
return options;
}

4
models/spring-ai-zhipuai/src/main/java/org/springframework/ai/zhipuai/ZhiPuAiChatModel.java
View File

@@ -177,7 +177,7 @@ public class ZhiPuAiChatModel extends AbstractToolCallSupport implements ChatMod
ChatResponse chatResponse = new ChatResponse(generations, from(completionEntity.getBody()));
if (isToolCall(chatResponse,
if (!isProxyToolCalls(prompt, this.defaultOptions) && isToolCall(chatResponse,
Set.of(ChatCompletionFinishReason.TOOL_CALLS.name(), ChatCompletionFinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, chatResponse);
// Recursively call the call method with the tool call message
@@ -241,7 +241,7 @@ public class ZhiPuAiChatModel extends AbstractToolCallSupport implements ChatMod
return chatResponse.flatMap(response -> {
if (isToolCall(response,
if (!isProxyToolCalls(prompt, this.defaultOptions) && isToolCall(response,
Set.of(ChatCompletionFinishReason.TOOL_CALLS.name(), ChatCompletionFinishReason.STOP.name()))) {
var toolCallConversation = handleToolCalls(prompt, response);
// Recursively call the stream method with the tool call message

25
models/spring-ai-zhipuai/src/main/java/org/springframework/ai/zhipuai/ZhiPuAiChatOptions.java
View File

@@ -123,6 +123,9 @@ public class ZhiPuAiChatOptions implements FunctionCallingOptions, ChatOptions {
@NestedConfigurationProperty
@JsonIgnore
private Set<String> functions = new HashSet<>();
@JsonIgnore
private Boolean proxyToolCalls;
// @formatter:on
public static Builder builder() {
@@ -208,6 +211,11 @@ public class ZhiPuAiChatOptions implements FunctionCallingOptions, ChatOptions {
return this;
}
public Builder withProxyToolCalls(Boolean proxyToolCalls) {
this.options.proxyToolCalls = proxyToolCalls;
return this;
}
public ZhiPuAiChatOptions build() {
return this.options;
}
@@ -346,6 +354,15 @@ public class ZhiPuAiChatOptions implements FunctionCallingOptions, ChatOptions {
return null;
}
@Override
public Boolean getProxyToolCalls() {
return this.proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
@Override
public int hashCode() {
final int prime = 31;
@@ -358,6 +375,7 @@ public class ZhiPuAiChatOptions implements FunctionCallingOptions, ChatOptions {
result = prime * result + ((tools == null) ? 0 : tools.hashCode());
result = prime * result + ((toolChoice == null) ? 0 : toolChoice.hashCode());
result = prime * result + ((user == null) ? 0 : user.hashCode());
result = prime * result + ((proxyToolCalls == null) ? 0 : proxyToolCalls.hashCode());
return result;
}
@@ -430,6 +448,12 @@ public class ZhiPuAiChatOptions implements FunctionCallingOptions, ChatOptions {
}
else if (!this.doSample.equals(other.doSample))
return false;
if (this.proxyToolCalls == null) {
if (other.proxyToolCalls != null)
return false;
}
else if (!this.proxyToolCalls.equals(other.proxyToolCalls))
return false;
return true;
}
@@ -452,6 +476,7 @@ public class ZhiPuAiChatOptions implements FunctionCallingOptions, ChatOptions {
.withDoSample(fromOptions.getDoSample())
.withFunctionCallbacks(fromOptions.getFunctionCallbacks())
.withFunctions(fromOptions.getFunctions())
.withProxyToolCalls(fromOptions.getProxyToolCalls())
.build();
}

1
spring-ai-core/src/main/java/org/springframework/ai/chat/client/AdvisedRequest.java
View File

@@ -64,6 +64,7 @@ public record AdvisedRequest(ChatModel chatModel, String userText, String system
builder.systemParams = from.systemParams;
builder.advisors = from.advisors;
builder.advisorParams = from.advisorParams;
builder.advisorParams = from.advisorParams;
return builder;
}

12
spring-ai-core/src/main/java/org/springframework/ai/chat/client/ChatClient.java
View File

@@ -74,9 +74,9 @@ public interface ChatClient {
ChatClientRequestSpec prompt();
ChatClientPromptRequestSpec prompt(String content);
ChatClientRequestSpec prompt(String content);
ChatClientPromptRequestSpec prompt(Prompt prompt);
ChatClientRequestSpec prompt(Prompt prompt);
/**
* Return a {@link ChatClient.Builder} to create a new {@link ChatClient} whose
@@ -159,14 +159,6 @@ public interface ChatClient {
}
interface ChatClientPromptRequestSpec {
CallPromptResponseSpec call();
StreamPromptResponseSpec stream();
}
interface CallPromptResponseSpec {
String content();

44
spring-ai-core/src/main/java/org/springframework/ai/chat/client/DefaultChatClient.java
View File

@@ -32,9 +32,9 @@ import org.springframework.ai.chat.client.advisor.DefaultAroundAdvisorChain;
import org.springframework.ai.chat.client.advisor.api.Advisor;
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor.StreamResponseMode;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservableHelper;
import org.springframework.ai.chat.client.observation.ChatClientObservationContext;
@@ -85,12 +85,9 @@ public class DefaultChatClient implements ChatClient {
private static final ChatClientObservationConvention DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION = new DefaultChatClientObservationConvention();
private final ChatModel chatModel;
private final DefaultChatClientRequestSpec defaultChatClientRequest;
public DefaultChatClient(ChatModel chatModel, DefaultChatClientRequestSpec defaultChatClientRequest) {
this.chatModel = chatModel;
public DefaultChatClient(DefaultChatClientRequestSpec defaultChatClientRequest) {
this.defaultChatClientRequest = defaultChatClientRequest;
}
@@ -100,13 +97,19 @@ public class DefaultChatClient implements ChatClient {
}
@Override
public ChatClientPromptRequestSpec prompt(String content) {
return new DefaultChatClientPromptRequestSpec(this.chatModel, new Prompt(content));
public ChatClientRequestSpec prompt(String content) {
return prompt(new Prompt(content));
}
@Override
public ChatClientPromptRequestSpec prompt(Prompt prompt) {
return new DefaultChatClientPromptRequestSpec(this.chatModel, prompt);
public ChatClientRequestSpec prompt(Prompt prompt) {
DefaultChatClientRequestSpec spec = new DefaultChatClientRequestSpec(this.defaultChatClientRequest);
spec.messages(prompt.getInstructions());
if (prompt.getOptions() != null) {
spec.options(prompt.getOptions());
}
return spec;
}
/**
@@ -997,25 +1000,4 @@ public class DefaultChatClient implements ChatClient {
}
public static class DefaultChatClientPromptRequestSpec implements ChatClientPromptRequestSpec {
private final ChatModel chatModel;
private final Prompt prompt;
public DefaultChatClientPromptRequestSpec(ChatModel chatModel, Prompt prompt) {
this.chatModel = chatModel;
this.prompt = prompt;
}
public CallPromptResponseSpec call() {
return new DefaultCallPromptResponseSpec(this.chatModel, this.prompt);
}
public StreamPromptResponseSpec stream() {
return new DefaultStreamPromptResponseSpec(this.chatModel, this.prompt);
}
}
}

5
spring-ai-core/src/main/java/org/springframework/ai/chat/client/DefaultChatClientBuilder.java
View File

@@ -50,8 +50,6 @@ public class DefaultChatClientBuilder implements Builder {
protected final DefaultChatClientRequestSpec defaultRequest;
private final ChatModel chatModel;
DefaultChatClientBuilder(ChatModel chatModel) {
this(chatModel, ObservationRegistry.NOOP, null);
}
@@ -60,14 +58,13 @@ public class DefaultChatClientBuilder implements Builder {
ChatClientObservationConvention customObservationConvention) {
Assert.notNull(chatModel, "the " + ChatModel.class.getName() + " must be non-null");
Assert.notNull(observationRegistry, "the " + ObservationRegistry.class.getName() + " must be non-null");
this.chatModel = chatModel;
this.defaultRequest = new DefaultChatClientRequestSpec(chatModel, "", Map.of(), "", Map.of(), List.of(),
List.of(), List.of(), List.of(), null, List.of(), Map.of(), observationRegistry,
customObservationConvention);
}
public ChatClient build() {
return new DefaultChatClient(this.chatModel, this.defaultRequest);
return new DefaultChatClient(this.defaultRequest);
}
public Builder defaultAdvisors(Advisor... advisor) {

37
spring-ai-core/src/main/java/org/springframework/ai/chat/model/AbstractToolCallSupport.java
View File

@@ -75,18 +75,18 @@ public abstract class AbstractToolCallSupport {
}
}
private static List<FunctionCallback> merge(FunctionCallingOptions funcitonOptions,
private static List<FunctionCallback> merge(FunctionCallingOptions functionOptions,
List<FunctionCallback> toolFunctionCallbacks) {
List<FunctionCallback> toolFunctionCallbacksCopy = new ArrayList<>();
if (!CollectionUtils.isEmpty(toolFunctionCallbacks)) {
toolFunctionCallbacksCopy.addAll(toolFunctionCallbacks);
}
if (!CollectionUtils.isEmpty(funcitonOptions.getFunctionCallbacks())) {
toolFunctionCallbacksCopy.addAll(funcitonOptions.getFunctionCallbacks());
if (!CollectionUtils.isEmpty(functionOptions.getFunctionCallbacks())) {
toolFunctionCallbacksCopy.addAll(functionOptions.getFunctionCallbacks());
// Make sure that that function callbacks are are registered directly to the
// functionCallbackRegister and not passed in the default options.
funcitonOptions.setFunctionCallbacks(List.of());
functionOptions.setFunctionCallbacks(List.of());
}
return toolFunctionCallbacksCopy;
}
@@ -220,6 +220,13 @@ public abstract class AbstractToolCallSupport {
return generations.stream().anyMatch(g -> isToolCall(g, toolCallFinishReasons));
}
/**
* Check if the generation is a tool call. The tool call finish reasons are used to
* determine if the generation is a tool call.
* @param generation the generation to check.
* @param toolCallFinishReasons the tool call finish reasons to check.
* @return true if the generation is a tool call, false otherwise.
*/
protected boolean isToolCall(Generation generation, Set<String> toolCallFinishReasons) {
var finishReason = (generation.getMetadata().getFinishReason() != null)
? generation.getMetadata().getFinishReason() : "";
@@ -229,4 +236,26 @@ public abstract class AbstractToolCallSupport {
.contains(finishReason.toLowerCase());
}
/**
* Check if the proxyToolCalls is enabled for the given prompt or the default tool
* call options. The prompt options take precedence over the default options. When the
* proxyToolCalls is enabled the ChatModel implementation will not handle the function
* calling internally. The tool call and tool response messages are exposed outside
* the ChatModel implementation.
* @param prompt the prompt to check.
* @param defaultOptions the default tool call options to check.
* @return true if the proxyToolCalls is enabled, false otherwise.
*/
protected boolean isProxyToolCalls(Prompt prompt, FunctionCallingOptions defaultOptions) {
if (prompt.getOptions() instanceof FunctionCallingOptions functionCallOptions
&& functionCallOptions.getProxyToolCalls() != null) {
return functionCallOptions.getProxyToolCalls();
}
else if (defaultOptions.getProxyToolCalls() != null) {
return defaultOptions.getProxyToolCalls();
}
return false;
}
}

2
spring-ai-core/src/main/java/org/springframework/ai/chat/prompt/Prompt.java
View File

@@ -50,7 +50,7 @@ public class Prompt implements ModelRequest<List<Message>> {
}
public Prompt(List<Message> messages) {
this.messages = messages;
this(messages, null);
}
public Prompt(String contents, ChatOptions chatOptions) {

10
spring-ai-core/src/main/java/org/springframework/ai/model/function/FunctionCallingOptions.java
View File

@@ -54,6 +54,16 @@ public interface FunctionCallingOptions {
*/
void setFunctions(Set<String> functions);
default Boolean getProxyToolCalls() {
return false;
}
default void setProxyToolCalls(Boolean proxyToolCalls) {
if (proxyToolCalls != null) {
throw new UnsupportedOperationException("Setting Proxy Tool Calls are not supported!");
}
}
/**
* @return Returns FunctionCallingOptionsBuilder to create a new instance of
* FunctionCallingOptions.

21
spring-ai-core/src/main/java/org/springframework/ai/model/function/FunctionCallingOptionsBuilder.java
View File

@@ -102,6 +102,11 @@ public class FunctionCallingOptionsBuilder {
return this;
}
public FunctionCallingOptionsBuilder withProxyToolCalls(Boolean proxyToolCalls) {
this.options.setProxyToolCalls(proxyToolCalls);
return this;
}
public PortableFunctionCallingOptions build() {
return this.options;
}
@@ -128,6 +133,12 @@ public class FunctionCallingOptionsBuilder {
private Double topP;
private Boolean proxyToolCalls = false;
public static FunctionCallingOptionsBuilder builder() {
return new FunctionCallingOptionsBuilder();
}
@Override
public List<FunctionCallback> getFunctionCallbacks() {
return this.functionCallbacks;
@@ -220,6 +231,15 @@ public class FunctionCallingOptionsBuilder {
this.topP = topP;
}
@Override
public Boolean getProxyToolCalls() {
return proxyToolCalls;
}
public void setProxyToolCalls(Boolean proxyToolCalls) {
this.proxyToolCalls = proxyToolCalls;
}
@Override
public ChatOptions copy() {
return new FunctionCallingOptionsBuilder().withModel(this.model)
@@ -232,6 +252,7 @@ public class FunctionCallingOptionsBuilder {
.withTopP(this.topP)
.withFunctions(this.functions)
.withFunctionCallbacks(this.functionCallbacks)
.withProxyToolCalls(this.proxyToolCalls)
.build();
}

166
spring-ai-core/src/main/java/org/springframework/ai/model/function/ToolCallHelper.java Normal file
View File

@@ -0,0 +1,166 @@
package org.springframework.ai.model.function;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.ToolResponseMessage;
import org.springframework.ai.chat.model.AbstractToolCallSupport;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.model.ModelOptionsUtils;
import org.springframework.ai.model.function.FunctionCallingOptionsBuilder.PortableFunctionCallingOptions;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
/**
* Helper class that reuses the {@link AbstractToolCallSupport} to implement the function
* call handling logic on the client side. Used when the withProxyToolCalls(true) option
* is enabled.
*/
public class ToolCallHelper extends AbstractToolCallSupport {
/**
* Helper used to provide only the function definition, without the actual function
* call implementation.
*/
public static record FunctionDefinition(String name, String description,
String inputTypeSchema) implements FunctionCallback {
@Override
public String getName() {
return this.name();
}
@Override
public String getDescription() {
return this.description();
}
@Override
public String getInputTypeSchema() {
return this.inputTypeSchema();
}
@Override
public String call(String functionInput) {
throw new UnsupportedOperationException(
"FunctionDefinition provides only metadata. It doesn't implement the call method.");
}
}
public ToolCallHelper() {
this(null, PortableFunctionCallingOptions.builder().build(), List.of());
}
public ToolCallHelper(FunctionCallbackContext functionCallbackContext,
FunctionCallingOptions functionCallingOptions, List<FunctionCallback> toolFunctionCallbacks) {
super(functionCallbackContext, functionCallingOptions, toolFunctionCallbacks);
}
@Override
public boolean isToolCall(ChatResponse chatResponse, Set<String> toolCallFinishReasons) {
return super.isToolCall(chatResponse, toolCallFinishReasons);
}
@Override
public List<Message> buildToolCallConversation(List<Message> previousMessages, AssistantMessage assistantMessage,
ToolResponseMessage toolResponseMessage) {
return super.buildToolCallConversation(previousMessages, assistantMessage, toolResponseMessage);
}
@Override
public List<Message> handleToolCalls(Prompt prompt, ChatResponse response) {
return super.handleToolCalls(prompt, response);
}
public Flux<ChatResponse> processStream(ChatModel chatModel, Prompt prompt, Set<String> finishReasons,
Function<AssistantMessage.ToolCall, String> customFunction) {
Flux<ChatResponse> chatResponses = chatModel.stream(prompt);
return chatResponses.flatMap(chatResponse -> {
boolean isToolCall = this.isToolCall(chatResponse, finishReasons);
if (isToolCall) {
Optional<Generation> toolCallGeneration = chatResponse.getResults()
.stream()
.filter(g -> !CollectionUtils.isEmpty(g.getOutput().getToolCalls()))
.findFirst();
AssistantMessage assistantMessage = toolCallGeneration.get().getOutput();
List<ToolResponseMessage.ToolResponse> toolResponses = new ArrayList<>();
for (AssistantMessage.ToolCall toolCall : assistantMessage.getToolCalls()) {
String functionResponse = customFunction.apply(toolCall);
toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolCall.name(),
ModelOptionsUtils.toJsonString(functionResponse)));
}
ToolResponseMessage toolMessageResponse = new ToolResponseMessage(toolResponses, Map.of());
List<Message> toolCallConversation = this.buildToolCallConversation(prompt.getInstructions(),
assistantMessage, toolMessageResponse);
var prompt2 = new Prompt(toolCallConversation, prompt.getOptions());
return processStream(chatModel, prompt2, finishReasons, customFunction);
}
return Flux.just(chatResponse);
});
}
public ChatResponse processCall(ChatModel chatModel, Prompt prompt, Set<String> finishReasons,
Function<AssistantMessage.ToolCall, String> customFunction) {
ChatResponse chatResponse = chatModel.call(prompt);
boolean isToolCall = this.isToolCall(chatResponse, finishReasons);
if (!isToolCall) {
return chatResponse;
}
Optional<Generation> toolCallGeneration = chatResponse.getResults()
.stream()
.filter(g -> !CollectionUtils.isEmpty(g.getOutput().getToolCalls()))
.findFirst();
AssistantMessage assistantMessage = toolCallGeneration.get().getOutput();
List<ToolResponseMessage.ToolResponse> toolResponses = new ArrayList<>();
for (AssistantMessage.ToolCall toolCall : assistantMessage.getToolCalls()) {
String functionResponse = customFunction.apply(toolCall);
toolResponses.add(new ToolResponseMessage.ToolResponse(toolCall.id(), toolCall.name(),
ModelOptionsUtils.toJsonString(functionResponse)));
}
ToolResponseMessage toolMessageResponse = new ToolResponseMessage(toolResponses, Map.of());
List<Message> toolCallConversation = this.buildToolCallConversation(prompt.getInstructions(), assistantMessage,
toolMessageResponse);
var prompt2 = new Prompt(toolCallConversation, prompt.getOptions());
return processCall(chatModel, prompt2, finishReasons, customFunction);
}
}

5
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc
View File

@@ -103,8 +103,9 @@ The prefix `spring.ai.anthropic.chat` is the property prefix that lets you confi
| spring.ai.anthropic.chat.options.stop-sequence | Custom text sequences that will cause the model to stop generating. Our models will normally stop when they have naturally completed their turn, which will result in a response stop_reason of "end_turn". If you want the model to stop generating when it encounters custom strings of text, you can use the stop_sequences parameter. If the model encounters one of the custom sequences, the response stop_reason value will be "stop_sequence" and the response stop_sequence value will contain the matched stop sequence. | -
| spring.ai.anthropic.chat.options.top-p | Use nucleus sampling. In nucleus sampling, we compute the cumulative distribution over all the options for each subsequent token in decreasing probability order and cut it off once it reaches a particular probability specified by top_p. You should either alter temperature or top_p, but not both. Recommended for advanced use cases only. You usually only need to use temperature. | -
| spring.ai.anthropic.chat.options.top-k | Only sample from the top K options for each subsequent token. Used to remove "long tail" low probability responses. Learn more technical details here. Recommended for advanced use cases only. You usually only need to use temperature. | -
| spring.ai.mistralai.chat.options.functions | List of functions, identified by their names, to enable for function calling in a single prompt requests. Functions with those names must exist in the functionCallbacks registry. | -
| spring.ai.mistralai.chat.options.functionCallbacks | MistralAI Tool Function Callbacks to register with the ChatModel. | -
| spring.ai.anthropic.chat.options.functions | List of functions, identified by their names, to enable for function calling in a single prompt requests. Functions with those names must exist in the functionCallbacks registry. | -
| spring.ai.anthropic.chat.options.functionCallbacks | Tool Function Callbacks to register with the ChatModel. | -
| spring.ai.anthropic.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====
TIP: All properties prefixed with `spring.ai.anthropic.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.

1
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/azure-openai-chat.adoc
View File

@@ -143,6 +143,7 @@ Deployments model name to provide as part of this completions request. | gpt-4o
| spring.ai.azure.openai.chat.options.presencePenalty | A value that influences the probability of generated tokens appearing based on their existing presence in generated text. Positive values will make tokens less likely to appear when they already exist and increase the model's likelihood to output new topics. | -
| spring.ai.azure.openai.chat.options.responseFormat | An object specifying the format that the model must output. Using `AzureOpenAiResponseFormat.JSON` enables JSON mode, which guarantees the message the model generates is valid JSON. Using AzureOpenAiResponseFormat.TEXT enables TEXT mode.| -
| spring.ai.azure.openai.chat.options.frequencyPenalty | A value that influences the probability of generated tokens appearing based on their cumulative frequency in generated text. Positive values will make tokens less likely to appear as their frequency increases and decrease the likelihood of the model repeating the same statements verbatim. | -
| spring.ai.azure.openai.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====
TIP: All properties prefixed with `spring.ai.azure.openai.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.

1
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/groq-chat.adoc
View File

@@ -125,6 +125,7 @@ The prefix `spring.ai.openai.chat` is the property prefix that lets you configur
| spring.ai.openai.chat.options.user | A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. | -
| spring.ai.openai.chat.options.functions | List of functions, identified by their names, to enable for function calling in a single prompt requests. Functions with those names must exist in the functionCallbacks registry. | -
| spring.ai.openai.chat.options.stream-usage | (For streaming only) Set to add an additional chunk with token usage statistics for the entire request. The `choices` field for this chunk is an empty array and all other chunks will also include a usage field, but with a null value. | false
| spring.ai.openai.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====
TIP: All properties prefixed with `spring.ai.openai.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.

1
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/mistralai-chat.adoc
View File

@@ -102,6 +102,7 @@ The prefix `spring.ai.mistralai.chat` is the property prefix that lets you confi
| spring.ai.mistralai.chat.options.toolChoice | Controls which (if any) function is called by the model. `none` means the model will not call a function and instead generates a message. `auto` means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type: "function", "function": {"name": "my_function"}}` forces the model to call that function. `none` is the default when no functions are present. `auto` is the default if functions are present. | -
| spring.ai.mistralai.chat.options.functions | List of functions, identified by their names, to enable for function calling in a single prompt requests. Functions with those names must exist in the functionCallbacks registry. | -
| spring.ai.mistralai.chat.options.functionCallbacks | Mistral AI Tool Function Callbacks to register with the ChatModel. | -
| spring.ai.mistralai.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====
NOTE: You can override the common `spring.ai.mistralai.base-url` and `spring.ai.mistralai.api-key` for the `ChatModel` and `EmbeddingModel` implementations.

1
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/nvidia-chat.adoc
View File

@@ -101,6 +101,7 @@ The prefix `spring.ai.openai.chat` is the property prefix that lets you configur
| spring.ai.openai.chat.options.user | A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. | -
| spring.ai.openai.chat.options.functions | List of functions, identified by their names, to enable for function calling in a single prompt requests. Functions with those names must exist in the functionCallbacks registry. | -
| spring.ai.openai.chat.options.stream-usage | (For streaming only) Set to add an additional chunk with token usage statistics for the entire request. The `choices` field for this chunk is an empty array and all other chunks will also include a usage field, but with a null value. | false
| spring.ai.openai.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====
TIP: All properties prefixed with `spring.ai.openai.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.

1
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc
View File

@@ -105,6 +105,7 @@ The remaining `options` properties are based on the link:https://github.com/olla
| spring.ai.ollama.chat.options.penalize-newline | - | true
| spring.ai.ollama.chat.options.stop | Sets the stop sequences to use. When this pattern is encountered the LLM will stop generating text and return. Multiple stop patterns may be set by specifying multiple separate stop parameters in a modelfile. | -
| spring.ai.ollama.chat.options.functions | List of functions, identified by their names, to enable for function calling in a single prompt requests. Functions with those names must exist in the functionCallbacks registry. | -
| spring.ai.ollama.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====
TIP: All properties prefixed with `spring.ai.ollama.chat.options` can be overridden at runtime by adding request-specific <<chat-options>> to the `Prompt` call.

1
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/openai-chat.adoc
View File

@@ -117,6 +117,7 @@ The `JSON_SCHEMA` type enables link:https://platform.openai.com/docs/guides/stru
| spring.ai.openai.chat.options.stream-usage | (For streaming only) Set to add an additional chunk with token usage statistics for the entire request. The `choices` field for this chunk is an empty array and all other chunks will also include a usage field, but with a null value. | false
| spring.ai.openai.chat.options.parallel-tool-calls | Whether to enable link:https://platform.openai.com/docs/guides/function-calling/parallel-function-calling[parallel function calling] during tool use. | true
| spring.ai.openai.chat.options.http-headers | Optional HTTP headers to be added to the chat completion request. To override the `api-key` you need to use an `Authorization` header key, and you have to prefix the key value with the `Bearer ` prefix. | -
| spring.ai.openai.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====
NOTE: You can override the common `spring.ai.openai.base-url` and `spring.ai.openai.api-key` for the `ChatModel` and `EmbeddingModel` implementations.

1
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/vertexai-gemini-chat.adoc
View File

@@ -78,6 +78,7 @@ The prefix `spring.ai.vertex.ai.gemini.chat` is the property prefix that lets yo
| spring.ai.vertex.ai.gemini.chat.options.frequencyPenalty | | -
| spring.ai.vertex.ai.gemini.chat.options.presencePenalty | | -
| spring.ai.vertex.ai.gemini.chat.options.functions | List of functions, identified by their names, to enable for function calling in a single prompt requests. Functions with those names must exist in the functionCallbacks registry. | -
| spring.ai.vertex.ai.gemini.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====

1
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/zhipuai-chat.adoc
View File

@@ -98,6 +98,7 @@ The prefix `spring.ai.zhipuai.chat` is the property prefix that lets you configu
| spring.ai.zhipuai.chat.options.user | A unique identifier representing your end-user, which can help ZhiPuAI to monitor and detect abuse. | -
| spring.ai.zhipuai.chat.options.requestId | The parameter is passed by the client and must ensure uniqueness. It is used to distinguish the unique identifier for each request. If the client does not provide it, the platform will generate it by default. | -
| spring.ai.zhipuai.chat.options.doSample | When do_sample is set to true, the sampling strategy is enabled. If do_sample is false, the sampling strategy parameters temperature and top_p will not take effect. | true
| spring.ai.zhipuai.chat.options.proxy-tool-calls | If true, the Spring AI will not handle the function calls internally, but will proxy them to the client. Then is the client's responsibility to handle the function calls, dispatch them to the appropriate function, and return the results. If false (the default), the Spring AI will handle the function calls internally. Applicable only for chat models with function calling support | false
|====
NOTE: You can override the common `spring.ai.zhipuai.base-url` and `spring.ai.zhipuai.api-key` for the `ChatModel` implementations.

9
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chatclient.adoc
View File

@@ -66,14 +66,13 @@ ChatClient chatClient = ChatClient.create(myChatModel);
== ChatClient Fluent API
The `ChatClient` fluent API allows you to create a prompt in three distinct ways using an overloaded `prompt` method to initiate the fluent API.
The `ChatClient` fluent API allows you to create a prompt in three distinct ways using an overloaded `prompt` method to initiate the fluent API:
* `prompt()`: This method with no arguments is the most flexible, allowing you to build up user, system, and other parts of the prompt using the fluent API.
* `prompt()`: This method with no arguments lets you start using the fluent API, allowing you to build up user, system, and other parts of the prompt.
* `prompt(Prompt prompt)`: This method accepts a `Prompt` argument, letting you pass in a `Prompt` instance that you have created using the Prompt's non-fluent APIs. The prompt passed to the model is 'sealed', meaning you can't modify it using fluent API methods.
* `prompt(String content)`: This is a convenience method similar to the previous overload. It takes the user's text content and creates a `Prompt` instance internally via `new Prompt(content)`.
* `prompt(Prompt prompt)`: This method accepts a `Prompt` argument, letting you pass in a `Prompt` instance that you have created using the Prompt's non-fluent APIs.
* `prompt(String content)`: This is a convenience method similar to the previous overload. It takes the user's text content.
== ChatClient Responses