Refactor Advisor Interfaces and add AroundAdvisor support

Refactoring and enhancements to the advisor functionality.

 New Advisor interfaces in the org.springframework.ai.chat.client.advisor.api package:
 - Advisor: Base interface for all advisor types.
 - RequestAdvisor: For advising on request data before execution.
 - ResponseAdvisor: For advising on response data after execution, with enhanced streaming modes.
 - CallAroundAdvisor and StreamAroundAdvisor: For around advice on synchronous and streaming requests respectively.
 - AroundAdvisorChain and DefaultAroundAdvisorChain: To manage chaining of around advisors.

 Advisor Chain and Prompt Generation:
 - Added the DefaultAroundAdvisorChain class to manage the sequence of advisors applied around chat model methods.
 - Adjusted the prompt generation (toPrompt) to integrate with the refactored AdvisedRequest object.

 Refactoring and Updates:
 - Replaced the deprecated RequestResponseAdvisor interface with RequestAdvisor and ResponseAdvisor across the spring-ai-core and test modules.
 - Updated the DefaultChatClient and related classes to use the new Advisor interface, improving modularity and consistency.
 - Refactored DefaultAdvisorSpec and DefaultChatClientRequestSpec to handle the new Advisor type, and revised advisor lists and methods accordingly.
 - Enhanced the handling of streaming responses, introducing StreamResponseMode for better control during streaming scenarios.
This commit is contained in:
Christian Tzolov
2024-08-29 15:07:48 +02:00
committed by Mark Pollack
parent 53721c642b
commit 28276d14a4
22 changed files with 954 additions and 255 deletions

View File

@@ -34,7 +34,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.model.Media;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatResponse;
@@ -45,6 +44,7 @@ import org.springframework.ai.chat.prompt.SystemPromptTemplate;
import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.ai.converter.ListOutputConverter;
import org.springframework.ai.converter.MapOutputConverter;
import org.springframework.ai.model.Media;
import org.springframework.ai.model.function.FunctionCallbackWrapper;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;

View File

@@ -16,6 +16,8 @@
package org.springframework.ai.openai.chat;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
@@ -26,11 +28,10 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.publisher.Flux;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.chat.client.RequestResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.ai.model.function.FunctionCallbackContext;
@@ -47,7 +48,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Description;
import org.springframework.core.ParameterizedTypeReference;
import static org.assertj.core.api.Assertions.assertThat;
import reactor.core.publisher.Flux;
/**
* @author Christian Tzolov
@@ -64,10 +65,14 @@ public class OpenAiPaymentTransactionIT {
record TransactionStatusResponse(String id, String status) {
}
private static class LoggingAdvisor implements RequestResponseAdvisor {
private static class LoggingAdvisor implements RequestAdvisor, ResponseAdvisor {
private final Logger logger = LoggerFactory.getLogger(LoggingAdvisor.class);
public String getName() {
return this.getClass().getSimpleName();
}
@Override
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
logger.info("System text: \n" + request.systemText());

View File

@@ -30,7 +30,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.chat.client.RequestResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.model.function.FunctionCallbackContext;
import org.springframework.ai.model.function.FunctionCallbackWrapper.Builder.SchemaType;
@@ -64,10 +65,15 @@ public class VertexAiGeminiPaymentTransactionIT {
record TransactionStatusResponse(String id, String status) {
}
private static class LoggingAdvisor implements RequestResponseAdvisor {
private static class LoggingAdvisor implements RequestAdvisor, ResponseAdvisor {
private final Logger logger = LoggerFactory.getLogger(LoggingAdvisor.class);
@Override
public String getName() {
return this.getClass().getSimpleName();
}
@Override
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
logger.info("System text: \n" + request.systemText());

View File

@@ -20,6 +20,7 @@ import java.util.List;
import java.util.Map;
import org.springframework.ai.model.Media;
import org.springframework.ai.chat.client.advisor.api.Advisor;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.prompt.ChatOptions;
@@ -46,7 +47,7 @@ import org.springframework.ai.model.function.FunctionCallback;
*/
public record AdvisedRequest(ChatModel chatModel, String userText, String systemText, ChatOptions chatOptions,
List<Media> media, List<String> functionNames, List<FunctionCallback> functionCallbacks, List<Message> messages,
Map<String, Object> userParams, Map<String, Object> systemParams, List<RequestResponseAdvisor> advisors,
Map<String, Object> userParams, Map<String, Object> systemParams, List<Advisor> advisors,
Map<String, Object> advisorParams) {
public static Builder from(AdvisedRequest from) {
@@ -92,7 +93,7 @@ public record AdvisedRequest(ChatModel chatModel, String userText, String system
private Map<String, Object> systemParams = Map.of();
private List<RequestResponseAdvisor> advisors = List.of();
private List<Advisor> advisors = List.of();
private Map<String, Object> advisorParams = Map.of();
@@ -146,7 +147,7 @@ public record AdvisedRequest(ChatModel chatModel, String userText, String system
return this;
}
public Builder withAdvisors(List<RequestResponseAdvisor> advisors) {
public Builder withAdvisors(List<Advisor> advisors) {
this.advisors = advisors;
return this;
}

View File

@@ -21,6 +21,7 @@ import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.springframework.ai.chat.client.advisor.api.Advisor;
import org.springframework.ai.chat.client.observation.ChatClientObservationConvention;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.model.ChatModel;
@@ -122,9 +123,9 @@ public interface ChatClient {
AdvisorSpec params(Map<String, Object> p);
AdvisorSpec advisors(RequestResponseAdvisor... advisors);
AdvisorSpec advisors(Advisor... advisors);
AdvisorSpec advisors(List<RequestResponseAdvisor> advisors);
AdvisorSpec advisors(List<Advisor> advisors);
}
@@ -192,9 +193,9 @@ public interface ChatClient {
ChatClientRequestSpec advisors(Consumer<AdvisorSpec> consumer);
ChatClientRequestSpec advisors(RequestResponseAdvisor... advisors);
ChatClientRequestSpec advisors(Advisor... advisors);
ChatClientRequestSpec advisors(List<RequestResponseAdvisor> advisors);
ChatClientRequestSpec advisors(List<Advisor> advisors);
ChatClientRequestSpec messages(Message... messages);
@@ -237,11 +238,11 @@ public interface ChatClient {
*/
interface Builder {
Builder defaultAdvisors(RequestResponseAdvisor... advisor);
Builder defaultAdvisors(Advisor... advisor);
Builder defaultAdvisors(Consumer<AdvisorSpec> advisorSpecConsumer);
Builder defaultAdvisors(List<RequestResponseAdvisor> advisors);
Builder defaultAdvisors(List<Advisor> advisors);
Builder defaultOptions(ChatOptions chatOptions);

View File

@@ -28,6 +28,14 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import org.springframework.ai.chat.client.advisor.DefaultAroundAdvisorChain;
import org.springframework.ai.chat.client.advisor.api.Advisor;
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor.StreamResponseMode;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservableHelper;
import org.springframework.ai.chat.client.observation.ChatClientObservationContext;
import org.springframework.ai.chat.client.observation.ChatClientObservationConvention;
@@ -38,6 +46,7 @@ import org.springframework.ai.chat.messages.SystemMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.MessageAggregator;
import org.springframework.ai.chat.model.StreamingChatModel;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.prompt.Prompt;
@@ -232,7 +241,7 @@ public class DefaultChatClient implements ChatClient {
public static class DefaultAdvisorSpec implements AdvisorSpec {
private final List<RequestResponseAdvisor> advisors = new ArrayList<>();
private final List<Advisor> advisors = new ArrayList<>();
private final Map<String, Object> params = new HashMap<>();
@@ -246,17 +255,17 @@ public class DefaultChatClient implements ChatClient {
return this;
}
public AdvisorSpec advisors(RequestResponseAdvisor... advisors) {
public AdvisorSpec advisors(Advisor... advisors) {
this.advisors.addAll(List.of(advisors));
return this;
}
public AdvisorSpec advisors(List<RequestResponseAdvisor> advisors) {
public AdvisorSpec advisors(List<Advisor> advisors) {
this.advisors.addAll(advisors);
return this;
}
public List<RequestResponseAdvisor> getAdvisors() {
public List<Advisor> getAdvisors() {
return advisors;
}
@@ -270,10 +279,7 @@ public class DefaultChatClient implements ChatClient {
private final DefaultChatClientRequestSpec request;
private final ChatModel chatModel;
public DefaultCallResponseSpec(ChatModel chatModel, DefaultChatClientRequestSpec request) {
this.chatModel = chatModel;
public DefaultCallResponseSpec(DefaultChatClientRequestSpec request) {
this.request = request;
}
@@ -330,8 +336,8 @@ public class DefaultChatClient implements ChatClient {
formatParam, false);
var observation = ChatClientObservationDocumentation.AI_CHAT_CLIENT.observation(
inputRequest.customObservationConvention, DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION,
() -> observationContext, inputRequest.observationRegistry);
inputRequest.getCustomObservationConvention(), DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION,
() -> observationContext, inputRequest.getObservationRegistry());
return observation.observe(() -> {
ChatResponse chatResponse = doGetChatResponse(inputRequest, formatParam, observation);
return chatResponse;
@@ -342,35 +348,37 @@ public class DefaultChatClient implements ChatClient {
private ChatResponse doGetChatResponse(DefaultChatClientRequestSpec inputRequestSpec, String formatParam,
Observation parentObservation) {
Map<String, Object> context = new ConcurrentHashMap<>();
context.putAll(inputRequestSpec.getAdvisorParams());
Map<String, Object> advisorContext = new ConcurrentHashMap<>();
if (StringUtils.hasText(formatParam)) {
advisorContext.put("formatParam", formatParam);
}
advisorContext.putAll(inputRequestSpec.getAdvisorParams());
DefaultChatClientRequestSpec advisedRequestSpec = inputRequestSpec;
// DefaultChatClientRequestSpec advisedRequestSpec = inputRequestSpec;
AdvisedRequest advisedRequest = toAdvisedRequest(inputRequestSpec);
if (!CollectionUtils.isEmpty(inputRequestSpec.advisors)) {
AdvisedRequest advisedRequest = toAdvisedRequest(inputRequestSpec);
// apply the advisors onRequest
var currentAdvisors = new ArrayList<>(inputRequestSpec.advisors);
for (RequestResponseAdvisor advisor : currentAdvisors) {
// Apply the Request advisors
var currentAdvisors = new ArrayList<>(
AdvisorObservableHelper.extractRequestAdvisors(inputRequestSpec.advisors));
for (RequestAdvisor advisor : currentAdvisors) {
advisedRequest = AdvisorObservableHelper.adviseRequest(parentObservation, advisor, advisedRequest,
context);
advisorContext);
}
advisedRequestSpec = toDefaultChatClientRequestSpec(advisedRequest,
inputRequestSpec.getObservationRegistry(), inputRequestSpec.getCustomObservationConvention());
}
var prompt = toPrompt(advisedRequestSpec, formatParam);
// Apply the around advisor chain that terminates with the, last, model call
// advisor.
ChatResponse advisedResponse = inputRequestSpec.aroundAdvisorChain.nextAroundCall(advisedRequest,
advisorContext);
var chatResponse = this.chatModel.call(prompt);
ChatResponse advisedResponse = chatResponse;
// apply the advisors on response
// Apply the Response advisors.
if (!CollectionUtils.isEmpty(inputRequestSpec.getAdvisors())) {
var currentAdvisors = new ArrayList<>(inputRequestSpec.getAdvisors());
for (RequestResponseAdvisor advisor : currentAdvisors) {
var currentAdvisors = new ArrayList<>(
AdvisorObservableHelper.extractResponseAdvisors(inputRequestSpec.getAdvisors()));
for (ResponseAdvisor advisor : currentAdvisors) {
advisedResponse = AdvisorObservableHelper.adviseResponse(parentObservation, advisor,
advisedResponse, context);
advisedResponse, advisorContext);
}
}
@@ -387,55 +395,51 @@ public class DefaultChatClient implements ChatClient {
}
static Prompt toPrompt(DefaultChatClientRequestSpec advisedRequest, String formatParam) {
private static Prompt toPrompt(AdvisedRequest advisedRequest, String formatParam) {
var messages = new ArrayList<Message>(advisedRequest.getMessages());
var messages = new ArrayList<Message>(advisedRequest.messages());
String processedSystemText = advisedRequest.getSystemText();
String processedSystemText = advisedRequest.systemText();
if (StringUtils.hasText(processedSystemText)) {
if (!CollectionUtils.isEmpty(advisedRequest.getSystemParams())) {
processedSystemText = new PromptTemplate(processedSystemText, advisedRequest.getSystemParams())
.render();
if (!CollectionUtils.isEmpty(advisedRequest.systemParams())) {
processedSystemText = new PromptTemplate(processedSystemText, advisedRequest.systemParams()).render();
}
messages.add(new SystemMessage(processedSystemText));
}
var processedUserText = StringUtils.hasText(formatParam)
? advisedRequest.getUserText() + System.lineSeparator() + "{spring_ai_soc_format}"
: advisedRequest.getUserText();
? advisedRequest.userText() + System.lineSeparator() + "{spring_ai_soc_format}"
: advisedRequest.userText();
if (StringUtils.hasText(processedUserText)) {
Map<String, Object> userParams = new HashMap<>(advisedRequest.getUserParams());
Map<String, Object> userParams = new HashMap<>(advisedRequest.userParams());
if (StringUtils.hasText(formatParam)) {
userParams.put("spring_ai_soc_format", formatParam);
}
if (!CollectionUtils.isEmpty(userParams)) {
processedUserText = new PromptTemplate(processedUserText, userParams).render();
}
messages.add(new UserMessage(processedUserText, advisedRequest.getMedia()));
messages.add(new UserMessage(processedUserText, advisedRequest.media()));
}
if (advisedRequest.getChatOptions() instanceof FunctionCallingOptions functionCallingOptions) {
if (!advisedRequest.getFunctionNames().isEmpty()) {
functionCallingOptions.setFunctions(new HashSet<>(advisedRequest.getFunctionNames()));
if (advisedRequest.chatOptions() instanceof FunctionCallingOptions functionCallingOptions) {
if (!advisedRequest.functionNames().isEmpty()) {
functionCallingOptions.setFunctions(new HashSet<>(advisedRequest.functionNames()));
}
if (!advisedRequest.getFunctionCallbacks().isEmpty()) {
functionCallingOptions.setFunctionCallbacks(advisedRequest.getFunctionCallbacks());
if (!advisedRequest.functionCallbacks().isEmpty()) {
functionCallingOptions.setFunctionCallbacks(advisedRequest.functionCallbacks());
}
}
return new Prompt(messages, advisedRequest.getChatOptions());
return new Prompt(messages, advisedRequest.chatOptions());
}
public static class DefaultStreamResponseSpec implements StreamResponseSpec {
private final DefaultChatClientRequestSpec request;
private final ChatModel chatModel;
public DefaultStreamResponseSpec(ChatModel chatModel, DefaultChatClientRequestSpec request) {
this.chatModel = chatModel;
public DefaultStreamResponseSpec(DefaultChatClientRequestSpec request) {
this.request = request;
}
@@ -446,20 +450,20 @@ public class DefaultChatClient implements ChatClient {
true);
Observation observation = ChatClientObservationDocumentation.AI_CHAT_CLIENT.observation(
inputRequest.customObservationConvention, DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION,
() -> observationContext, inputRequest.observationRegistry);
inputRequest.getCustomObservationConvention(), DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION,
() -> observationContext, inputRequest.getObservationRegistry());
observation.parentObservation(contextView.getOrDefault(ObservationThreadLocalAccessor.KEY, null))
.start();
// @formatter:off
return doGetFluxChatResponse(inputRequest, observation)
.doOnError(observation::error)
.doFinally(s -> {
observation.stop();
})
.contextWrite(ctx -> ctx.put(ObservationThreadLocalAccessor.KEY, observation));
// @formatter:on
return doGetFluxChatResponse(inputRequest, observation)
.doOnError(observation::error)
.doFinally(s -> {
observation.stop();
})
.contextWrite(ctx -> ctx.put(ObservationThreadLocalAccessor.KEY, observation));
// @formatter:on
});
}
@@ -473,33 +477,93 @@ public class DefaultChatClient implements ChatClient {
var reqWithContext = new AdvisedRequestWithContext(toAdvisedRequest(inputRequest), advisorContext);
return Flux.fromIterable(inputRequest.advisors)
return Flux.fromIterable(AdvisorObservableHelper.extractRequestAdvisors(inputRequest.advisors))
.transformDeferredContextual((f, ctx) -> f
// This allows us to call blocking code in reduce
.publishOn(Schedulers.boundedElastic())
.reduce(reqWithContext, (rwc, advisor) -> {
// Apply the Request advisors
AdvisedRequest advisedRequest = AdvisorObservableHelper.adviseRequest(parentObservation,
advisor, rwc.request, rwc.advisorContext);
return new AdvisedRequestWithContext(advisedRequest, rwc.advisorContext);
}))
.single()
.flatMapMany(rwc -> {
DefaultChatClientRequestSpec advisedRequest = toDefaultChatClientRequestSpec(rwc.request,
inputRequest.getObservationRegistry(), inputRequest.getCustomObservationConvention());
var prompt = toPrompt(advisedRequest, null);
// Apply the around advisor chain that terminates with the, last,
// model call advisor.
Flux<ChatResponse> advisedResponse = inputRequest.aroundAdvisorChain.nextAroundStream(rwc.request,
rwc.advisorContext);
Flux<ChatResponse> fluxChatResponse = this.chatModel.stream(prompt);
Flux<ChatResponse> advisedResponse = fluxChatResponse;
// apply the advisors on response
// Apply the Response advisors
if (!CollectionUtils.isEmpty(inputRequest.getAdvisors())) {
var currentAdvisors = new ArrayList<>(inputRequest.getAdvisors());
for (RequestResponseAdvisor advisor : currentAdvisors) {
advisedResponse = AdvisorObservableHelper.adviseResponse(parentObservation, advisor,
advisedResponse, advisorContext);
var responseAdvisors = new ArrayList<>(
AdvisorObservableHelper.extractResponseAdvisors(inputRequest.getAdvisors()));
List<ResponseAdvisor> perElementResponseAdvisors = responseAdvisors.stream()
.filter(a -> a.getStreamResponseMode() == StreamResponseMode.PER_ELEMENT)
.toList();
List<ResponseAdvisor> onFinishElementResponseAdvisors = responseAdvisors.stream()
.filter(a -> a.getStreamResponseMode() == StreamResponseMode.ON_FINISH_ELEMENT)
.toList();
// PER_ELEMENT and ON_FINISH_ELEMENT
advisedResponse = advisedResponse.map(response -> {
// PER_ELEMENT
if (!CollectionUtils.isEmpty(perElementResponseAdvisors)) {
for (ResponseAdvisor advisor : perElementResponseAdvisors) {
response = AdvisorObservableHelper.adviseResponse(parentObservation, advisor,
response, rwc.advisorContext);
}
}
// ON_FINISH_ELEMENT
if (!CollectionUtils.isEmpty(onFinishElementResponseAdvisors)) {
for (ResponseAdvisor advisor : onFinishElementResponseAdvisors) {
boolean withFinishReason = response.getResults()
.stream()
.filter(result -> result != null && result.getMetadata() != null
&& StringUtils.hasText(result.getMetadata().getFinishReason()))
.findFirst()
.isPresent();
if (withFinishReason) {
response = AdvisorObservableHelper.adviseResponse(parentObservation, advisor,
response, advisorContext);
}
}
}
return response;
});
// CUSTOM
// TODO: how to pass the parentObservation to the custom response
// advisor?
List<ResponseAdvisor> customResponseAdvisors = responseAdvisors.stream()
.filter(a -> a.getStreamResponseMode() == StreamResponseMode.CUSTOM)
.toList();
if (!CollectionUtils.isEmpty(customResponseAdvisors)) {
for (ResponseAdvisor advisor : customResponseAdvisors) {
advisedResponse = advisor.adviseResponse(advisedResponse, rwc.advisorContext);
}
}
// AGGREGATE
List<ResponseAdvisor> aggregateResponseAdvisors = responseAdvisors.stream()
.filter(a -> a.getStreamResponseMode() == StreamResponseMode.AGGREGATE)
.toList();
if (!CollectionUtils.isEmpty(aggregateResponseAdvisors)) {
advisedResponse = new MessageAggregator().aggregate(advisedResponse, chatResponse -> {
for (ResponseAdvisor advisor : aggregateResponseAdvisors) {
AdvisorObservableHelper.adviseResponse(parentObservation, advisor, chatResponse,
advisorContext);
}
});
}
}
return advisedResponse;
});
@@ -547,52 +611,58 @@ public class DefaultChatClient implements ChatClient {
private final Map<String, Object> systemParams = new HashMap<>();
private final List<RequestResponseAdvisor> advisors = new ArrayList<>();
private final List<Advisor> advisors = new ArrayList<>();
private final Map<String, Object> advisorParams = new HashMap<>();
private final DefaultAroundAdvisorChain aroundAdvisorChain;
public AroundAdvisorChain getAroundAdvisorChain() {
return this.aroundAdvisorChain;
}
private ObservationRegistry getObservationRegistry() {
return observationRegistry;
return this.observationRegistry;
}
private ChatClientObservationConvention getCustomObservationConvention() {
return customObservationConvention;
return this.customObservationConvention;
}
public String getUserText() {
return userText;
return this.userText;
}
public Map<String, Object> getUserParams() {
return userParams;
return this.userParams;
}
public String getSystemText() {
return systemText;
return this.systemText;
}
public Map<String, Object> getSystemParams() {
return systemParams;
return this.systemParams;
}
public ChatOptions getChatOptions() {
return chatOptions;
return this.chatOptions;
}
public List<RequestResponseAdvisor> getAdvisors() {
return advisors;
public List<Advisor> getAdvisors() {
return this.advisors;
}
public Map<String, Object> getAdvisorParams() {
return advisorParams;
return this.advisorParams;
}
public List<Message> getMessages() {
return messages;
return this.messages;
}
public List<Media> getMedia() {
return media;
return this.media;
}
public List<String> getFunctionNames() {
@@ -600,7 +670,7 @@ public class DefaultChatClient implements ChatClient {
}
public List<FunctionCallback> getFunctionCallbacks() {
return functionCallbacks;
return this.functionCallbacks;
}
/* copy constructor */
@@ -613,8 +683,8 @@ public class DefaultChatClient implements ChatClient {
public DefaultChatClientRequestSpec(ChatModel chatModel, String userText, Map<String, Object> userParams,
String systemText, Map<String, Object> systemParams, List<FunctionCallback> functionCallbacks,
List<Message> messages, List<String> functionNames, List<Media> media, ChatOptions chatOptions,
List<RequestResponseAdvisor> advisors, Map<String, Object> advisorParams,
ObservationRegistry observationRegistry, ChatClientObservationConvention customObservationConvention) {
List<Advisor> advisors, Map<String, Object> advisorParams, ObservationRegistry observationRegistry,
ChatClientObservationConvention customObservationConvention) {
this.chatModel = chatModel;
this.chatOptions = chatOptions != null ? chatOptions.copy()
@@ -633,6 +703,40 @@ public class DefaultChatClient implements ChatClient {
this.advisorParams.putAll(advisorParams);
this.observationRegistry = observationRegistry;
this.customObservationConvention = customObservationConvention;
// @formatter:off
this.aroundAdvisorChain = DefaultAroundAdvisorChain.builder(observationRegistry)
// At the stack bottom add the non-streaming and streaming model call advisors.
// They play the role of the last advisor in the around advisor chain.
.push(new CallAroundAdvisor() {
@Override
public String getName() {
return CallAroundAdvisor.class.getSimpleName();
}
@Override
public ChatResponse aroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
AroundAdvisorChain chain) {
String formatParam = (String) adviceContext.get("formatParam");
return chatModel.call(toPrompt(advisedRequest, formatParam));
}
})
.push(new StreamAroundAdvisor() {
@Override
public String getName() {
return StreamAroundAdvisor.class.getSimpleName();
}
@Override
public Flux<ChatResponse> aroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
AroundAdvisorChain chain) {
return chatModel.stream(toPrompt(advisedRequest, null));
}
})
.pushAll(this.advisors)
.build();
// @formatter:on
}
/**
@@ -662,18 +766,21 @@ public class DefaultChatClient implements ChatClient {
consumer.accept(as);
this.advisorParams.putAll(as.getParams());
this.advisors.addAll(as.getAdvisors());
this.aroundAdvisorChain.pushAll(as.getAdvisors());
return this;
}
public ChatClientRequestSpec advisors(RequestResponseAdvisor... advisors) {
public ChatClientRequestSpec advisors(Advisor... advisors) {
Assert.notNull(advisors, "the advisors must be non-null");
this.advisors.addAll(Arrays.asList(advisors));
this.aroundAdvisorChain.pushAll(Arrays.asList(advisors));
return this;
}
public ChatClientRequestSpec advisors(List<RequestResponseAdvisor> advisors) {
public ChatClientRequestSpec advisors(List<Advisor> advisors) {
Assert.notNull(advisors, "the advisors must be non-null");
this.advisors.addAll(advisors);
this.aroundAdvisorChain.pushAll(advisors);
return this;
}
@@ -797,11 +904,11 @@ public class DefaultChatClient implements ChatClient {
}
public CallResponseSpec call() {
return new DefaultCallResponseSpec(chatModel, this);
return new DefaultCallResponseSpec(this);
}
public StreamResponseSpec stream() {
return new DefaultStreamResponseSpec(chatModel, this);
return new DefaultStreamResponseSpec(this);
}
}
@@ -906,4 +1013,4 @@ public class DefaultChatClient implements ChatClient {
}
}
}

View File

@@ -26,6 +26,7 @@ import org.springframework.ai.chat.client.ChatClient.Builder;
import org.springframework.ai.chat.client.ChatClient.PromptSystemSpec;
import org.springframework.ai.chat.client.ChatClient.PromptUserSpec;
import org.springframework.ai.chat.client.DefaultChatClient.DefaultChatClientRequestSpec;
import org.springframework.ai.chat.client.advisor.api.Advisor;
import org.springframework.ai.chat.client.observation.ChatClientObservationConvention;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.prompt.ChatOptions;
@@ -69,7 +70,7 @@ public class DefaultChatClientBuilder implements Builder {
return new DefaultChatClient(this.chatModel, this.defaultRequest);
}
public Builder defaultAdvisors(RequestResponseAdvisor... advisor) {
public Builder defaultAdvisors(Advisor... advisor) {
this.defaultRequest.advisors(advisor);
return this;
}
@@ -79,7 +80,7 @@ public class DefaultChatClientBuilder implements Builder {
return this;
}
public Builder defaultAdvisors(List<RequestResponseAdvisor> advisors) {
public Builder defaultAdvisors(List<Advisor> advisors) {
this.defaultRequest.advisors(advisors);
return this;
}

View File

@@ -16,120 +16,41 @@
package org.springframework.ai.chat.client;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import java.util.Map;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.MessageAggregator;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.util.StringUtils;
import reactor.core.publisher.Flux;
/**
* Advisor called before and after the {@link ChatModel#call(Prompt)} and
* {@link ChatModel#stream(Prompt)} methods calls. The {@link ChatClient} maintains a
* chain of advisors with chared execution context.
* chain of advisors with shared advise context.
*
* @deprecated since 1.0.0 please use {@link RequestAdvisor}, {@link ResponseAdvisor}
* instead.
* @author Christian Tzolov
* @since 1.0.0
*/
public interface RequestResponseAdvisor {
@Deprecated
public interface RequestResponseAdvisor extends RequestAdvisor, ResponseAdvisor {
public enum StreamResponseMode {
/**
* The sync advisor will be called for each response chunk (e.g. on each Flux
* item).
*/
PER_CHUNK,
/**
* The sync advisor is called only on chunks that contain a finish reason. Usually
* the last chunk in the stream.
*/
ON_FINISH_REASON,
/**
* The sync advisor is called only once after the stream is completed and an
* aggregated response is computed. Note that at that stage the advisor can not
* modify the response, but only observe it and react on the aggregated response.
*/
AGGREGATE,
/**
* Delegates to the stream advisor implementation.
*/
CUSTOM;
}
default StreamResponseMode getStreamResponseMode() {
return StreamResponseMode.CUSTOM;
}
/**
* @return the advisor name.
*/
@Override
default String getName() {
return this.getClass().getSimpleName();
}
/**
* @param request the {@link AdvisedRequest} data to be advised. Represents the row
* {@link ChatClient.ChatClientRequestSpec} data before sealed into a {@link Prompt}.
* @param context the shared data between the advisors in the chain. It is shared
* between all request and response advising points of all advisors in the chain.
* @return the advised {@link AdvisedRequest}.
*/
default AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
@Override
default AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> adviseContext) {
return request;
}
/**
* @param response the {@link ChatResponse} data to be advised. Represents the row
* {@link ChatResponse} data after the {@link ChatModel#call(Prompt)} method is
* called.
* @param context the shared data between the advisors in the chain. It is shared
* between all request and response advising points of all advisors in the chain.
* @return the advised {@link ChatResponse}.
*/
default ChatResponse adviseResponse(ChatResponse response, Map<String, Object> context) {
@Override
default ChatResponse adviseResponse(ChatResponse response, Map<String, Object> adviseContext) {
return response;
}
/**
* @param fluxResponse the streaming {@link ChatResponse} data to be advised.
* Represents the row {@link ChatResponse} stream data after the
* {@link ChatModel#stream(Prompt)} method is called.
* @param context the shared data between the advisors in the chain. It is shared
* between all request and response advising points of all advisors in the chain.
* @return the advised {@link ChatResponse} flux.
*/
default Flux<ChatResponse> adviseResponse(Flux<ChatResponse> fluxResponse, Map<String, Object> context) {
if (this.getStreamResponseMode() == StreamResponseMode.PER_CHUNK) {
return fluxResponse.map(chatResponse -> this.adviseResponse(chatResponse, context));
}
else if (this.getStreamResponseMode() == StreamResponseMode.AGGREGATE) {
return new MessageAggregator().aggregate(fluxResponse, chatResponse -> {
this.adviseResponse(chatResponse, context);
});
}
else if (this.getStreamResponseMode() == StreamResponseMode.ON_FINISH_REASON) {
return fluxResponse.map(chatResponse -> {
boolean withFinishReason = chatResponse.getResults()
.stream()
.filter(result -> result != null && result.getMetadata() != null
&& StringUtils.hasText(result.getMetadata().getFinishReason()))
.findFirst()
.isPresent();
if (withFinishReason) {
return this.adviseResponse(chatResponse, context);
}
return chatResponse;
});
}
return fluxResponse;
}
}
}

View File

@@ -18,7 +18,8 @@ package org.springframework.ai.chat.client.advisor;
import java.util.Map;
import org.springframework.ai.chat.client.RequestResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.util.Assert;
/**
@@ -28,7 +29,7 @@ import org.springframework.util.Assert;
* @author Christian Tzolov
* @since 1.0.0 M1
*/
public abstract class AbstractChatMemoryAdvisor<T> implements RequestResponseAdvisor {
public abstract class AbstractChatMemoryAdvisor<T> implements RequestAdvisor, ResponseAdvisor {
public static final String CHAT_MEMORY_CONVERSATION_ID_KEY = "chat_memory_conversation_id";
@@ -60,8 +61,8 @@ public abstract class AbstractChatMemoryAdvisor<T> implements RequestResponseAdv
}
@Override
public StreamResponseMode getStreamResponseMode() {
return StreamResponseMode.AGGREGATE;
public String getName() {
return this.getClass().getSimpleName();
}
protected T getChatMemoryStore() {

View File

@@ -0,0 +1,158 @@
package org.springframework.ai.chat.client.advisor;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import java.util.Map;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.advisor.api.Advisor;
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservableHelper;
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservationContext;
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservationDocumentation;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.util.Assert;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.contextpropagation.ObservationThreadLocalAccessor;
import reactor.core.publisher.Flux;
public class DefaultAroundAdvisorChain implements AroundAdvisorChain {
private final Deque<CallAroundAdvisor> callAroundAdvisors;
private final Deque<StreamAroundAdvisor> streamAroundAdvisors;
private final ObservationRegistry observationRegistry;
public DefaultAroundAdvisorChain(ObservationRegistry observationRegistry) {
this(observationRegistry, new ArrayDeque<CallAroundAdvisor>(), new ArrayDeque<StreamAroundAdvisor>());
}
public DefaultAroundAdvisorChain(CallAroundAdvisor aroundAdvisor, ObservationRegistry observationRegistry) {
this(observationRegistry, new ArrayDeque<CallAroundAdvisor>(), new ArrayDeque<StreamAroundAdvisor>());
this.push(aroundAdvisor);
}
public DefaultAroundAdvisorChain(ObservationRegistry observationRegistry,
Deque<CallAroundAdvisor> callAroundAdvisors, Deque<StreamAroundAdvisor> streamAroundAdvisors) {
Assert.notNull(callAroundAdvisors, "the callAroundAdvisors must be non-null");
this.observationRegistry = observationRegistry;
this.callAroundAdvisors = callAroundAdvisors;
this.streamAroundAdvisors = streamAroundAdvisors;
}
public DefaultAroundAdvisorChain(ObservationRegistry observationRegistry, List<Advisor> advisors) {
this(observationRegistry);
Assert.notNull(advisors, "the advisors must be non-null");
advisors.forEach(this::push);
}
public void pushAll(List<? extends Advisor> advisors) {
Assert.notNull(advisors, "the advisors must be non-null");
advisors.forEach(this::push);
}
public void push(Advisor aroundAdvisor) {
Assert.notNull(aroundAdvisor, "the aroundAdvisor must be non-null");
if (aroundAdvisor instanceof CallAroundAdvisor callAroundAdvisor) {
this.callAroundAdvisors.push(callAroundAdvisor);
}
// Note: the advisor can implement both the CallAroundAdvisor and
// StreamAroundAdvisor.
if (aroundAdvisor instanceof StreamAroundAdvisor streamAroundAdvisor) {
this.streamAroundAdvisors.push(streamAroundAdvisor);
}
}
@Override
public ChatResponse nextAroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext) {
if (this.callAroundAdvisors.isEmpty()) {
throw new IllegalStateException("No AroundAdvisor available to execute");
}
var advisor = this.callAroundAdvisors.pop();
var observationContext = AdvisorObservationContext.builder()
.withAdvisorName(advisor.getName())
.withAdvisorType(AdvisorObservationContext.Type.AROUND)
.withAdvisedRequest(advisedRequest)
.withAdvisorRequestContext(adviceContext)
.build();
return AdvisorObservationDocumentation.AI_ADVISOR
.observation(null, AdvisorObservableHelper.DEFAULT_OBSERVATION_CONVENTION, () -> observationContext,
this.observationRegistry)
.observe(() -> advisor.aroundCall(advisedRequest, adviceContext, this));
}
@Override
public Flux<ChatResponse> nextAroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext) {
return Flux.deferContextual(contextView -> {
if (this.streamAroundAdvisors.isEmpty()) {
return Flux.error(new IllegalStateException("No AroundAdvisor available to execute"));
}
var advisor = this.streamAroundAdvisors.pop();
AdvisorObservationContext observationContext = AdvisorObservationContext.builder()
.withAdvisorName(advisor.getName())
.withAdvisorType(AdvisorObservationContext.Type.AROUND)
.withAdvisedRequest(advisedRequest)
.withAdvisorRequestContext(adviceContext)
.build();
var observation = AdvisorObservationDocumentation.AI_ADVISOR.observation(null,
AdvisorObservableHelper.DEFAULT_OBSERVATION_CONVENTION, () -> observationContext,
this.observationRegistry);
observation.parentObservation(contextView.getOrDefault(ObservationThreadLocalAccessor.KEY, null)).start();
return advisor.aroundStream(advisedRequest, adviceContext, this)
.doOnError(observation::error)
.doFinally(s -> {
observation.stop();
})
.contextWrite(ctx -> ctx.put(ObservationThreadLocalAccessor.KEY, observation));
});
}
public static Builder builder(ObservationRegistry observationRegistry) {
return new Builder(observationRegistry);
}
public static class Builder {
private final DefaultAroundAdvisorChain aroundAdvisorChain;
public Builder(ObservationRegistry observationRegistry) {
this.aroundAdvisorChain = new DefaultAroundAdvisorChain(observationRegistry);
}
public Builder push(Advisor aroundAdvisor) {
Assert.notNull(aroundAdvisor, "the aroundAdvisor must be non-null");
this.aroundAdvisorChain.push(aroundAdvisor);
return this;
}
public Builder pushAll(List<Advisor> aroundAdvisors) {
Assert.notNull(aroundAdvisors, "the aroundAdvisors must be non-null");
this.aroundAdvisorChain.pushAll(aroundAdvisors);
return this;
}
public DefaultAroundAdvisorChain build() {
return this.aroundAdvisorChain;
}
}
}

View File

@@ -22,7 +22,8 @@ import java.util.Map;
import java.util.stream.Collectors;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.RequestResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.document.Document;
import org.springframework.ai.model.Content;
@@ -40,7 +41,7 @@ import org.springframework.util.StringUtils;
* @author Christian Tzolov
* @since 1.0.0
*/
public class QuestionAnswerAdvisor implements RequestResponseAdvisor {
public class QuestionAnswerAdvisor implements RequestAdvisor, ResponseAdvisor {
private static final String DEFAULT_USER_TEXT_ADVISE = """
Context information is below.
@@ -91,6 +92,11 @@ public class QuestionAnswerAdvisor implements RequestResponseAdvisor {
this.userTextAdvise = userTextAdvise;
}
@Override
public String getName() {
return this.getClass().getSimpleName();
}
@Override
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
@@ -140,9 +146,4 @@ public class QuestionAnswerAdvisor implements RequestResponseAdvisor {
}
@Override
public StreamResponseMode getStreamResponseMode() {
return StreamResponseMode.ON_FINISH_REASON;
}
}
}

View File

@@ -21,7 +21,8 @@ import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.RequestResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.model.ModelOptionsUtils;
@@ -30,7 +31,7 @@ import org.springframework.ai.model.ModelOptionsUtils;
*
* @author Christian Tzolov
*/
public class SimpleLoggerAdvisor implements RequestResponseAdvisor {
public class SimpleLoggerAdvisor implements RequestAdvisor, ResponseAdvisor {
private static final Logger logger = LoggerFactory.getLogger(SimpleLoggerAdvisor.class);
@@ -56,6 +57,11 @@ public class SimpleLoggerAdvisor implements RequestResponseAdvisor {
this.responseToString = responseToString;
}
@Override
public String getName() {
return this.getClass().getSimpleName();
}
@Override
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
logger.debug("request: {}", this.requestToString.apply(request));
@@ -73,9 +79,4 @@ public class SimpleLoggerAdvisor implements RequestResponseAdvisor {
return SimpleLoggerAdvisor.class.getSimpleName();
}
@Override
public StreamResponseMode getStreamResponseMode() {
return StreamResponseMode.AGGREGATE;
}
}

View File

@@ -0,0 +1,35 @@
/*
* Copyright 2024 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.chat.client.advisor.api;
import org.springframework.ai.chat.client.advisor.DefaultAroundAdvisorChain;
/**
* Parent advisor interface for all advisors.
*
* @author Christian Tzolov
* @since 1.0.0
* @see {@link RequestAdvisor}, {@link ResponseAdvisor}, {@link CallAroundAdvisor},
* {@link StreamAroundAdvisor}, {@link DefaultAroundAdvisorChain}
*/
public interface Advisor {
/**
* @return the advisor name.
*/
String getName();
}

View File

@@ -0,0 +1,16 @@
package org.springframework.ai.chat.client.advisor.api;
import java.util.Map;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.model.ChatResponse;
import reactor.core.publisher.Flux;
public interface AroundAdvisorChain {
ChatResponse nextAroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext);
Flux<ChatResponse> nextAroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext);
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright 2024 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.chat.client.advisor.api;
import java.util.Map;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.model.ChatResponse;
/**
* @author Christian Tzolov
* @since 1.0.0
*/
public interface CallAroundAdvisor extends Advisor {
/**
* Around advice that wraps the {@link ChatModel#call(Prompt)} method.
* @param advisedRequest the advised request
* @param adviceContext the advice context
* @param chain the advisor chain
* @return the response
*/
ChatResponse aroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext, AroundAdvisorChain chain);
}

View File

@@ -0,0 +1,46 @@
/*
* Copyright 2024-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.chat.client.advisor.api;
import java.util.Map;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.prompt.Prompt;
/**
* Advisor called before the {@link ChatModel#call(Prompt)} and
* {@link ChatModel#stream(Prompt)} methods are called. The {@link ChatClient} maintains a
* chain of advisors with shared advise context.
*
* @author Christian Tzolov
* @since 1.0.0
*/
public interface RequestAdvisor extends Advisor {
/**
* @param request the {@link AdvisedRequest} data to be advised. Represents the row
* {@link ChatClient.ChatClientRequestSpec} data before sealed into a {@link Prompt}.
* @param adviseContext the shared data between the advisors in the chain. It is
* shared between all request and response advising points of all advisors in the
* chain.
* @return the advised {@link AdvisedRequest}.
*/
AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> adviseContext);
}

View File

@@ -0,0 +1,97 @@
/*
* Copyright 2024-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.chat.client.advisor.api;
import java.util.Map;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import reactor.core.publisher.Flux;
/**
* Advisor called after the {@link ChatModel#call(Prompt)} (or
* {@link ChatModel#stream(Prompt)}) method call. The {@link ChatClient} maintains a chain
* of advisors with shared advise context.
*
* @author Christian Tzolov
* @since 1.0.0
*/
public interface ResponseAdvisor extends Advisor {
/**
* @param response the {@link ChatResponse} data to be advised. Represents the row
* {@link ChatResponse} data after the {@link ChatModel#call(Prompt)} method is
* called.
* @param adviseContext the shared data between the advisors in the chain. It is
* shared between all request and response advising points of all advisors in the
* chain.
* @return the advised {@link ChatResponse}.
*/
ChatResponse adviseResponse(ChatResponse response, Map<String, Object> adviseContext);
/**
* Different modes of advising the streaming responses.
*/
public enum StreamResponseMode {
/**
* Called for each response element in the Flux. The response advisor can modify
* the elements before they are returned to the client.
*/
PER_ELEMENT,
/**
* Called only on Flux elements that contain a finish reason. Usually the last
* element in the Flux. The response advisor can modify the elements before they
* are returned to the client.
*/
ON_FINISH_ELEMENT,
/**
* Called only once after all Flux elements have been consumed. All elements are
* merged into a single ChatResponse element and provided to the response advisor
* to process. <br/>
* Mind that at that stage the response advisor can not longer modify the response
* returned to the client.
*/
AGGREGATE,
/**
* Delegates to the stream advisor implementation.
*/
CUSTOM;
}
default StreamResponseMode getStreamResponseMode() {
return StreamResponseMode.AGGREGATE;
}
/**
* @param fluxResponse the streaming {@link ChatResponse} data to be advised.
* Represents the row {@link ChatResponse} stream data after the
* {@link ChatModel#stream(Prompt)} method is called.
* @param adviseContext the shared data between the advisors in the chain. It is
* shared between all request and response advising points of all advisors in the
* chain.
* @return the advised {@link ChatResponse} flux.
*/
default Flux<ChatResponse> adviseResponse(Flux<ChatResponse> fluxResponse, Map<String, Object> adviseContext) {
return fluxResponse;
}
}

View File

@@ -0,0 +1,41 @@
/*
* Copyright 2024 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.chat.client.advisor.api;
import java.util.Map;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.model.ChatResponse;
import reactor.core.publisher.Flux;
/**
* @author Christian Tzolov
* @since 1.0.0
*/
public interface StreamAroundAdvisor extends Advisor {
/**
* Around advice that wraps the invocation of the advised request.
* @param advisedRequest
* @param adviceContext
* @param chain the chain of advisors to execute
* @return the result of the advised request
*/
Flux<ChatResponse> aroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
AroundAdvisorChain chain);
}

View File

@@ -0,0 +1,149 @@
/*
* Copyright 2024 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.chat.client.advisor.around;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.MessageType;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.model.MessageAggregator;
import org.springframework.ai.document.Document;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
/**
* @author Christian Tzolov
* @since 1.0.0
*/
public class CacheAroundAdvisor implements CallAroundAdvisor, StreamAroundAdvisor {
private final VectorStore vectorStore;
private static final String DOCUMENT_METADATA_ADVISOR_CACHE_TAG = "advisorCacheDocument";
private static final String DOCUMENT_METADATA_ADVISOR_CACHE_RESPONSE = "advisorCacheResponse";
public CacheAroundAdvisor(VectorStore vectorStore) {
this.vectorStore = vectorStore;
}
public String getName() {
return this.getClass().getSimpleName();
}
@Override
public ChatResponse aroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
AroundAdvisorChain chain) {
var cachedResponseOption = getCacheEntry(advisedRequest, adviceContext);
if (cachedResponseOption.isPresent()) {
return cachedResponseOption.get();
}
ChatResponse chatResponse = chain.nextAroundCall(advisedRequest, adviceContext);
saveCacheEntry(advisedRequest.userText(), chatResponse);
return chatResponse;
}
@Override
public Flux<ChatResponse> aroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
AroundAdvisorChain chain) {
var cachedResponseOption = getCacheEntry(advisedRequest, adviceContext);
if (cachedResponseOption.isPresent()) {
return Flux.just(cachedResponseOption.get());
}
Flux<ChatResponse> fluxChatResponse = chain.nextAroundStream(advisedRequest, adviceContext);
return new MessageAggregator().aggregate(fluxChatResponse, chatResponse -> {
saveCacheEntry(advisedRequest.userText(), chatResponse);
});
}
private void saveCacheEntry(String userQuestion, ChatResponse chatResponse) {
List<Message> assistantMessages = chatResponse.getResults().stream().map(g -> (Message) g.getOutput()).toList();
if (!CollectionUtils.isEmpty(assistantMessages)) {
this.vectorStore.add(toDocuments(userQuestion, assistantMessages));
}
}
private Optional<ChatResponse> getCacheEntry(AdvisedRequest advisedRequest, Map<String, Object> adviceContext) {
// TODO: convert into pompty first or at least materialize the user params.
String userText = advisedRequest.userText();
// @formatter:off
var searchRequest = SearchRequest.query(userText)
.withSimilarityThreshold(0.95)
.withTopK(1)
.withFilterExpression("'"+ DOCUMENT_METADATA_ADVISOR_CACHE_TAG + "' == 'true'");
// @formatter:on
List<Document> doc = vectorStore.similaritySearch(searchRequest);
// return cached response
return CollectionUtils.isEmpty(doc) ? Optional.empty() : Optional.of(fromDocument(doc.get(0)));
}
private ChatResponse fromDocument(Document doc) {
if (!doc.getMetadata().containsKey(DOCUMENT_METADATA_ADVISOR_CACHE_RESPONSE)) {
throw new IllegalStateException("The document is missing the cache response metadata!");
}
String cachedResponse = "" + doc.getMetadata().get(DOCUMENT_METADATA_ADVISOR_CACHE_RESPONSE);
return ChatResponse.builder()
.withGenerations(List.of(new Generation(new AssistantMessage(cachedResponse, Map.of()),
ChatGenerationMetadata.from("STOP", null))))
.build();
}
private List<Document> toDocuments(String userQuestion, List<Message> messages) {
List<Document> docs = messages.stream()
.filter(m -> m.getMessageType() == MessageType.ASSISTANT)
.map(message -> {
var metadata = new HashMap<>(message.getMetadata() != null ? message.getMetadata() : new HashMap<>());
metadata.put(DOCUMENT_METADATA_ADVISOR_CACHE_TAG, "true");
metadata.put(DOCUMENT_METADATA_ADVISOR_CACHE_RESPONSE, message.getContent());
// TODO: Pehaps we need to serialize the message metadata to the document
return new Document(userQuestion, metadata);
})
.toList();
return docs;
}
}

View File

@@ -0,0 +1,59 @@
package org.springframework.ai.chat.client.advisor.around;
import java.util.List;
import java.util.Map;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
/**
* A {@link CallAroundAdvisor} and {@link StreamAroundAdvisor} that filters out the
* response if the user input contains any of the sensitive words.
*
* @author Christian Tzolov
* @since 1.0.0
*/
public class SafeGuardAroundAdvisor implements CallAroundAdvisor, StreamAroundAdvisor {
private final List<String> sensitiveWords;
public SafeGuardAroundAdvisor(List<String> sensitiveWords) {
this.sensitiveWords = sensitiveWords;
}
public String getName() {
return this.getClass().getSimpleName();
}
@Override
public ChatResponse aroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
AroundAdvisorChain chain) {
if (!CollectionUtils.isEmpty(this.sensitiveWords)
&& sensitiveWords.stream().anyMatch(w -> advisedRequest.userText().contains(w))) {
return ChatResponse.builder().withGenerations(List.of()).build();
}
return chain.nextAroundCall(advisedRequest, adviceContext);
}
@Override
public Flux<ChatResponse> aroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
AroundAdvisorChain chain) {
if (!CollectionUtils.isEmpty(this.sensitiveWords)
&& sensitiveWords.stream().anyMatch(w -> advisedRequest.userText().contains(w))) {
return Flux.empty();
}
return chain.nextAroundStream(advisedRequest, adviceContext);
}
}

View File

@@ -15,17 +15,19 @@
*/
package org.springframework.ai.chat.client.advisor.observation;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.RequestResponseAdvisor;
import org.springframework.ai.chat.client.RequestResponseAdvisor.StreamResponseMode;
import org.springframework.ai.chat.client.advisor.api.Advisor;
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.MessageAggregator;
import org.springframework.util.StringUtils;
import org.springframework.util.CollectionUtils;
import io.micrometer.observation.Observation;
import reactor.core.publisher.Flux;
/**
* @author Christian Tzolov
@@ -33,9 +35,9 @@ import reactor.core.publisher.Flux;
*/
public abstract class AdvisorObservableHelper {
private static final AdvisorObservationConvention DEFAULT_OBSERVATION_CONVENTION = new DefaultAdvisorObservationConvention();
public static final AdvisorObservationConvention DEFAULT_OBSERVATION_CONVENTION = new DefaultAdvisorObservationConvention();
public static AdvisedRequest adviseRequest(Observation parentObservation, RequestResponseAdvisor advisor,
public static AdvisedRequest adviseRequest(Observation parentObservation, RequestAdvisor advisor,
AdvisedRequest advisedRequest, Map<String, Object> advisorContext) {
var observationContext = AdvisorObservationContext.builder()
@@ -52,7 +54,7 @@ public abstract class AdvisorObservableHelper {
.observe(() -> advisor.adviseRequest(advisedRequest, advisorContext));
}
public static ChatResponse adviseResponse(Observation parentObservation, RequestResponseAdvisor advisor,
public static ChatResponse adviseResponse(Observation parentObservation, ResponseAdvisor advisor,
ChatResponse response, Map<String, Object> advisorContext) {
var observationContext = AdvisorObservationContext.builder()
@@ -68,35 +70,34 @@ public abstract class AdvisorObservableHelper {
.observe(() -> advisor.adviseResponse(response, advisorContext));
}
public static Flux<ChatResponse> adviseResponse(Observation parentObservation, RequestResponseAdvisor advisor,
Flux<ChatResponse> fluxResponse, Map<String, Object> advisorContext) {
public static List<RequestAdvisor> extractRequestAdvisors(List<Advisor> advisors) {
return advisors.stream()
.filter(advisor -> advisor instanceof RequestAdvisor)
.map(a -> (RequestAdvisor) a)
.toList();
}
if (advisor.getStreamResponseMode() == StreamResponseMode.PER_CHUNK) {
return fluxResponse
.map(chatResponse -> adviseResponse(parentObservation, advisor, chatResponse, advisorContext));
}
else if (advisor.getStreamResponseMode() == StreamResponseMode.AGGREGATE) {
return new MessageAggregator().aggregate(fluxResponse, chatResponse -> {
adviseResponse(parentObservation, advisor, chatResponse, advisorContext);
});
}
else if (advisor.getStreamResponseMode() == StreamResponseMode.ON_FINISH_REASON) {
return fluxResponse.map(chatResponse -> {
boolean withFinishReason = chatResponse.getResults()
.stream()
.filter(result -> result != null && result.getMetadata() != null
&& StringUtils.hasText(result.getMetadata().getFinishReason()))
.findFirst()
.isPresent();
/**
* Extracts the {@link ResponseAdvisor} instances from the given list of advisors and
* returns them in reverse order.
* @param advisors list of all registered advisor types.
* @return the list of {@link ResponseAdvisor} instances in reverse order.
*/
public static List<ResponseAdvisor> extractResponseAdvisors(List<Advisor> advisors) {
if (withFinishReason) {
return adviseResponse(parentObservation, advisor, chatResponse, advisorContext);
}
return chatResponse;
});
var list = advisors.stream()
.filter(advisor -> advisor instanceof ResponseAdvisor)
.map(a -> (ResponseAdvisor) a)
.toList();
// reverse the list
if (CollectionUtils.isEmpty(list)) {
return list;
}
return advisor.adviseResponse(fluxResponse, advisorContext);
var reversedList = new ArrayList<>(list);
Collections.reverse(reversedList);
return Collections.unmodifiableList(reversedList);
}
}

View File

@@ -25,12 +25,14 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.ai.chat.client.AdvisedRequest;
import org.springframework.ai.chat.client.DefaultChatClient.DefaultChatClientRequestSpec;
import org.springframework.ai.chat.client.RequestResponseAdvisor;
import org.springframework.ai.chat.client.observation.ChatClientObservationDocumentation.HighCardinalityKeyNames;
import org.springframework.ai.chat.client.observation.ChatClientObservationDocumentation.LowCardinalityKeyNames;
import org.springframework.ai.chat.metadata.Usage;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.model.function.FunctionCallback;
import io.micrometer.common.KeyValue;
@@ -93,6 +95,17 @@ class DefaultChatClientObservationConventionTests {
public String getName() {
return name;
}
@Override
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
return request;
}
@Override
public ChatResponse adviseResponse(ChatResponse response, Map<String, Object> adviseContext) {
return response;
}
};
}