Refactor Advisor Interfaces and add AroundAdvisor support
Refactoring and enhancements to the advisor functionality. New Advisor interfaces in the org.springframework.ai.chat.client.advisor.api package: - Advisor: Base interface for all advisor types. - RequestAdvisor: For advising on request data before execution. - ResponseAdvisor: For advising on response data after execution, with enhanced streaming modes. - CallAroundAdvisor and StreamAroundAdvisor: For around advice on synchronous and streaming requests respectively. - AroundAdvisorChain and DefaultAroundAdvisorChain: To manage chaining of around advisors. Advisor Chain and Prompt Generation: - Added the DefaultAroundAdvisorChain class to manage the sequence of advisors applied around chat model methods. - Adjusted the prompt generation (toPrompt) to integrate with the refactored AdvisedRequest object. Refactoring and Updates: - Replaced the deprecated RequestResponseAdvisor interface with RequestAdvisor and ResponseAdvisor across the spring-ai-core and test modules. - Updated the DefaultChatClient and related classes to use the new Advisor interface, improving modularity and consistency. - Refactored DefaultAdvisorSpec and DefaultChatClientRequestSpec to handle the new Advisor type, and revised advisor lists and methods accordingly. - Enhanced the handling of streaming responses, introducing StreamResponseMode for better control during streaming scenarios.
This commit is contained in:
committed by
Mark Pollack
parent
53721c642b
commit
28276d14a4
@@ -34,7 +34,6 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.model.Media;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
@@ -45,6 +44,7 @@ import org.springframework.ai.chat.prompt.SystemPromptTemplate;
|
||||
import org.springframework.ai.converter.BeanOutputConverter;
|
||||
import org.springframework.ai.converter.ListOutputConverter;
|
||||
import org.springframework.ai.converter.MapOutputConverter;
|
||||
import org.springframework.ai.model.Media;
|
||||
import org.springframework.ai.model.function.FunctionCallbackWrapper;
|
||||
import org.springframework.ai.openai.OpenAiChatModel;
|
||||
import org.springframework.ai.openai.OpenAiChatOptions;
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
|
||||
package org.springframework.ai.openai.chat;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
@@ -26,11 +28,10 @@ import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.client.RequestResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.converter.BeanOutputConverter;
|
||||
import org.springframework.ai.model.function.FunctionCallbackContext;
|
||||
@@ -47,7 +48,7 @@ import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Description;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
@@ -64,10 +65,14 @@ public class OpenAiPaymentTransactionIT {
|
||||
record TransactionStatusResponse(String id, String status) {
|
||||
}
|
||||
|
||||
private static class LoggingAdvisor implements RequestResponseAdvisor {
|
||||
private static class LoggingAdvisor implements RequestAdvisor, ResponseAdvisor {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(LoggingAdvisor.class);
|
||||
|
||||
public String getName() {
|
||||
return this.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
|
||||
logger.info("System text: \n" + request.systemText());
|
||||
|
||||
@@ -30,7 +30,8 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.client.RequestResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.model.function.FunctionCallbackContext;
|
||||
import org.springframework.ai.model.function.FunctionCallbackWrapper.Builder.SchemaType;
|
||||
@@ -64,10 +65,15 @@ public class VertexAiGeminiPaymentTransactionIT {
|
||||
record TransactionStatusResponse(String id, String status) {
|
||||
}
|
||||
|
||||
private static class LoggingAdvisor implements RequestResponseAdvisor {
|
||||
private static class LoggingAdvisor implements RequestAdvisor, ResponseAdvisor {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(LoggingAdvisor.class);
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return this.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
|
||||
logger.info("System text: \n" + request.systemText());
|
||||
|
||||
@@ -20,6 +20,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.model.Media;
|
||||
import org.springframework.ai.chat.client.advisor.api.Advisor;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||
@@ -46,7 +47,7 @@ import org.springframework.ai.model.function.FunctionCallback;
|
||||
*/
|
||||
public record AdvisedRequest(ChatModel chatModel, String userText, String systemText, ChatOptions chatOptions,
|
||||
List<Media> media, List<String> functionNames, List<FunctionCallback> functionCallbacks, List<Message> messages,
|
||||
Map<String, Object> userParams, Map<String, Object> systemParams, List<RequestResponseAdvisor> advisors,
|
||||
Map<String, Object> userParams, Map<String, Object> systemParams, List<Advisor> advisors,
|
||||
Map<String, Object> advisorParams) {
|
||||
|
||||
public static Builder from(AdvisedRequest from) {
|
||||
@@ -92,7 +93,7 @@ public record AdvisedRequest(ChatModel chatModel, String userText, String system
|
||||
|
||||
private Map<String, Object> systemParams = Map.of();
|
||||
|
||||
private List<RequestResponseAdvisor> advisors = List.of();
|
||||
private List<Advisor> advisors = List.of();
|
||||
|
||||
private Map<String, Object> advisorParams = Map.of();
|
||||
|
||||
@@ -146,7 +147,7 @@ public record AdvisedRequest(ChatModel chatModel, String userText, String system
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withAdvisors(List<RequestResponseAdvisor> advisors) {
|
||||
public Builder withAdvisors(List<Advisor> advisors) {
|
||||
this.advisors = advisors;
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import org.springframework.ai.chat.client.advisor.api.Advisor;
|
||||
import org.springframework.ai.chat.client.observation.ChatClientObservationConvention;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
@@ -122,9 +123,9 @@ public interface ChatClient {
|
||||
|
||||
AdvisorSpec params(Map<String, Object> p);
|
||||
|
||||
AdvisorSpec advisors(RequestResponseAdvisor... advisors);
|
||||
AdvisorSpec advisors(Advisor... advisors);
|
||||
|
||||
AdvisorSpec advisors(List<RequestResponseAdvisor> advisors);
|
||||
AdvisorSpec advisors(List<Advisor> advisors);
|
||||
|
||||
}
|
||||
|
||||
@@ -192,9 +193,9 @@ public interface ChatClient {
|
||||
|
||||
ChatClientRequestSpec advisors(Consumer<AdvisorSpec> consumer);
|
||||
|
||||
ChatClientRequestSpec advisors(RequestResponseAdvisor... advisors);
|
||||
ChatClientRequestSpec advisors(Advisor... advisors);
|
||||
|
||||
ChatClientRequestSpec advisors(List<RequestResponseAdvisor> advisors);
|
||||
ChatClientRequestSpec advisors(List<Advisor> advisors);
|
||||
|
||||
ChatClientRequestSpec messages(Message... messages);
|
||||
|
||||
@@ -237,11 +238,11 @@ public interface ChatClient {
|
||||
*/
|
||||
interface Builder {
|
||||
|
||||
Builder defaultAdvisors(RequestResponseAdvisor... advisor);
|
||||
Builder defaultAdvisors(Advisor... advisor);
|
||||
|
||||
Builder defaultAdvisors(Consumer<AdvisorSpec> advisorSpecConsumer);
|
||||
|
||||
Builder defaultAdvisors(List<RequestResponseAdvisor> advisors);
|
||||
Builder defaultAdvisors(List<Advisor> advisors);
|
||||
|
||||
Builder defaultOptions(ChatOptions chatOptions);
|
||||
|
||||
|
||||
@@ -28,6 +28,14 @@ import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import org.springframework.ai.chat.client.advisor.DefaultAroundAdvisorChain;
|
||||
import org.springframework.ai.chat.client.advisor.api.Advisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
|
||||
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor.StreamResponseMode;
|
||||
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservableHelper;
|
||||
import org.springframework.ai.chat.client.observation.ChatClientObservationContext;
|
||||
import org.springframework.ai.chat.client.observation.ChatClientObservationConvention;
|
||||
@@ -38,6 +46,7 @@ import org.springframework.ai.chat.messages.SystemMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.MessageAggregator;
|
||||
import org.springframework.ai.chat.model.StreamingChatModel;
|
||||
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
@@ -232,7 +241,7 @@ public class DefaultChatClient implements ChatClient {
|
||||
|
||||
public static class DefaultAdvisorSpec implements AdvisorSpec {
|
||||
|
||||
private final List<RequestResponseAdvisor> advisors = new ArrayList<>();
|
||||
private final List<Advisor> advisors = new ArrayList<>();
|
||||
|
||||
private final Map<String, Object> params = new HashMap<>();
|
||||
|
||||
@@ -246,17 +255,17 @@ public class DefaultChatClient implements ChatClient {
|
||||
return this;
|
||||
}
|
||||
|
||||
public AdvisorSpec advisors(RequestResponseAdvisor... advisors) {
|
||||
public AdvisorSpec advisors(Advisor... advisors) {
|
||||
this.advisors.addAll(List.of(advisors));
|
||||
return this;
|
||||
}
|
||||
|
||||
public AdvisorSpec advisors(List<RequestResponseAdvisor> advisors) {
|
||||
public AdvisorSpec advisors(List<Advisor> advisors) {
|
||||
this.advisors.addAll(advisors);
|
||||
return this;
|
||||
}
|
||||
|
||||
public List<RequestResponseAdvisor> getAdvisors() {
|
||||
public List<Advisor> getAdvisors() {
|
||||
return advisors;
|
||||
}
|
||||
|
||||
@@ -270,10 +279,7 @@ public class DefaultChatClient implements ChatClient {
|
||||
|
||||
private final DefaultChatClientRequestSpec request;
|
||||
|
||||
private final ChatModel chatModel;
|
||||
|
||||
public DefaultCallResponseSpec(ChatModel chatModel, DefaultChatClientRequestSpec request) {
|
||||
this.chatModel = chatModel;
|
||||
public DefaultCallResponseSpec(DefaultChatClientRequestSpec request) {
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
@@ -330,8 +336,8 @@ public class DefaultChatClient implements ChatClient {
|
||||
formatParam, false);
|
||||
|
||||
var observation = ChatClientObservationDocumentation.AI_CHAT_CLIENT.observation(
|
||||
inputRequest.customObservationConvention, DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION,
|
||||
() -> observationContext, inputRequest.observationRegistry);
|
||||
inputRequest.getCustomObservationConvention(), DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION,
|
||||
() -> observationContext, inputRequest.getObservationRegistry());
|
||||
return observation.observe(() -> {
|
||||
ChatResponse chatResponse = doGetChatResponse(inputRequest, formatParam, observation);
|
||||
return chatResponse;
|
||||
@@ -342,35 +348,37 @@ public class DefaultChatClient implements ChatClient {
|
||||
private ChatResponse doGetChatResponse(DefaultChatClientRequestSpec inputRequestSpec, String formatParam,
|
||||
Observation parentObservation) {
|
||||
|
||||
Map<String, Object> context = new ConcurrentHashMap<>();
|
||||
context.putAll(inputRequestSpec.getAdvisorParams());
|
||||
Map<String, Object> advisorContext = new ConcurrentHashMap<>();
|
||||
if (StringUtils.hasText(formatParam)) {
|
||||
advisorContext.put("formatParam", formatParam);
|
||||
}
|
||||
advisorContext.putAll(inputRequestSpec.getAdvisorParams());
|
||||
|
||||
DefaultChatClientRequestSpec advisedRequestSpec = inputRequestSpec;
|
||||
// DefaultChatClientRequestSpec advisedRequestSpec = inputRequestSpec;
|
||||
AdvisedRequest advisedRequest = toAdvisedRequest(inputRequestSpec);
|
||||
if (!CollectionUtils.isEmpty(inputRequestSpec.advisors)) {
|
||||
|
||||
AdvisedRequest advisedRequest = toAdvisedRequest(inputRequestSpec);
|
||||
|
||||
// apply the advisors onRequest
|
||||
var currentAdvisors = new ArrayList<>(inputRequestSpec.advisors);
|
||||
for (RequestResponseAdvisor advisor : currentAdvisors) {
|
||||
// Apply the Request advisors
|
||||
var currentAdvisors = new ArrayList<>(
|
||||
AdvisorObservableHelper.extractRequestAdvisors(inputRequestSpec.advisors));
|
||||
for (RequestAdvisor advisor : currentAdvisors) {
|
||||
advisedRequest = AdvisorObservableHelper.adviseRequest(parentObservation, advisor, advisedRequest,
|
||||
context);
|
||||
advisorContext);
|
||||
}
|
||||
advisedRequestSpec = toDefaultChatClientRequestSpec(advisedRequest,
|
||||
inputRequestSpec.getObservationRegistry(), inputRequestSpec.getCustomObservationConvention());
|
||||
}
|
||||
|
||||
var prompt = toPrompt(advisedRequestSpec, formatParam);
|
||||
// Apply the around advisor chain that terminates with the, last, model call
|
||||
// advisor.
|
||||
ChatResponse advisedResponse = inputRequestSpec.aroundAdvisorChain.nextAroundCall(advisedRequest,
|
||||
advisorContext);
|
||||
|
||||
var chatResponse = this.chatModel.call(prompt);
|
||||
|
||||
ChatResponse advisedResponse = chatResponse;
|
||||
// apply the advisors on response
|
||||
// Apply the Response advisors.
|
||||
if (!CollectionUtils.isEmpty(inputRequestSpec.getAdvisors())) {
|
||||
var currentAdvisors = new ArrayList<>(inputRequestSpec.getAdvisors());
|
||||
for (RequestResponseAdvisor advisor : currentAdvisors) {
|
||||
var currentAdvisors = new ArrayList<>(
|
||||
AdvisorObservableHelper.extractResponseAdvisors(inputRequestSpec.getAdvisors()));
|
||||
for (ResponseAdvisor advisor : currentAdvisors) {
|
||||
advisedResponse = AdvisorObservableHelper.adviseResponse(parentObservation, advisor,
|
||||
advisedResponse, context);
|
||||
advisedResponse, advisorContext);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -387,55 +395,51 @@ public class DefaultChatClient implements ChatClient {
|
||||
|
||||
}
|
||||
|
||||
static Prompt toPrompt(DefaultChatClientRequestSpec advisedRequest, String formatParam) {
|
||||
private static Prompt toPrompt(AdvisedRequest advisedRequest, String formatParam) {
|
||||
|
||||
var messages = new ArrayList<Message>(advisedRequest.getMessages());
|
||||
var messages = new ArrayList<Message>(advisedRequest.messages());
|
||||
|
||||
String processedSystemText = advisedRequest.getSystemText();
|
||||
String processedSystemText = advisedRequest.systemText();
|
||||
if (StringUtils.hasText(processedSystemText)) {
|
||||
if (!CollectionUtils.isEmpty(advisedRequest.getSystemParams())) {
|
||||
processedSystemText = new PromptTemplate(processedSystemText, advisedRequest.getSystemParams())
|
||||
.render();
|
||||
if (!CollectionUtils.isEmpty(advisedRequest.systemParams())) {
|
||||
processedSystemText = new PromptTemplate(processedSystemText, advisedRequest.systemParams()).render();
|
||||
}
|
||||
messages.add(new SystemMessage(processedSystemText));
|
||||
}
|
||||
|
||||
var processedUserText = StringUtils.hasText(formatParam)
|
||||
? advisedRequest.getUserText() + System.lineSeparator() + "{spring_ai_soc_format}"
|
||||
: advisedRequest.getUserText();
|
||||
? advisedRequest.userText() + System.lineSeparator() + "{spring_ai_soc_format}"
|
||||
: advisedRequest.userText();
|
||||
|
||||
if (StringUtils.hasText(processedUserText)) {
|
||||
|
||||
Map<String, Object> userParams = new HashMap<>(advisedRequest.getUserParams());
|
||||
Map<String, Object> userParams = new HashMap<>(advisedRequest.userParams());
|
||||
if (StringUtils.hasText(formatParam)) {
|
||||
userParams.put("spring_ai_soc_format", formatParam);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(userParams)) {
|
||||
processedUserText = new PromptTemplate(processedUserText, userParams).render();
|
||||
}
|
||||
messages.add(new UserMessage(processedUserText, advisedRequest.getMedia()));
|
||||
messages.add(new UserMessage(processedUserText, advisedRequest.media()));
|
||||
}
|
||||
|
||||
if (advisedRequest.getChatOptions() instanceof FunctionCallingOptions functionCallingOptions) {
|
||||
if (!advisedRequest.getFunctionNames().isEmpty()) {
|
||||
functionCallingOptions.setFunctions(new HashSet<>(advisedRequest.getFunctionNames()));
|
||||
if (advisedRequest.chatOptions() instanceof FunctionCallingOptions functionCallingOptions) {
|
||||
if (!advisedRequest.functionNames().isEmpty()) {
|
||||
functionCallingOptions.setFunctions(new HashSet<>(advisedRequest.functionNames()));
|
||||
}
|
||||
if (!advisedRequest.getFunctionCallbacks().isEmpty()) {
|
||||
functionCallingOptions.setFunctionCallbacks(advisedRequest.getFunctionCallbacks());
|
||||
if (!advisedRequest.functionCallbacks().isEmpty()) {
|
||||
functionCallingOptions.setFunctionCallbacks(advisedRequest.functionCallbacks());
|
||||
}
|
||||
}
|
||||
|
||||
return new Prompt(messages, advisedRequest.getChatOptions());
|
||||
return new Prompt(messages, advisedRequest.chatOptions());
|
||||
}
|
||||
|
||||
public static class DefaultStreamResponseSpec implements StreamResponseSpec {
|
||||
|
||||
private final DefaultChatClientRequestSpec request;
|
||||
|
||||
private final ChatModel chatModel;
|
||||
|
||||
public DefaultStreamResponseSpec(ChatModel chatModel, DefaultChatClientRequestSpec request) {
|
||||
this.chatModel = chatModel;
|
||||
public DefaultStreamResponseSpec(DefaultChatClientRequestSpec request) {
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
@@ -446,20 +450,20 @@ public class DefaultChatClient implements ChatClient {
|
||||
true);
|
||||
|
||||
Observation observation = ChatClientObservationDocumentation.AI_CHAT_CLIENT.observation(
|
||||
inputRequest.customObservationConvention, DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION,
|
||||
() -> observationContext, inputRequest.observationRegistry);
|
||||
inputRequest.getCustomObservationConvention(), DEFAULT_CHAT_CLIENT_OBSERVATION_CONVENTION,
|
||||
() -> observationContext, inputRequest.getObservationRegistry());
|
||||
|
||||
observation.parentObservation(contextView.getOrDefault(ObservationThreadLocalAccessor.KEY, null))
|
||||
.start();
|
||||
|
||||
// @formatter:off
|
||||
return doGetFluxChatResponse(inputRequest, observation)
|
||||
.doOnError(observation::error)
|
||||
.doFinally(s -> {
|
||||
observation.stop();
|
||||
})
|
||||
.contextWrite(ctx -> ctx.put(ObservationThreadLocalAccessor.KEY, observation));
|
||||
// @formatter:on
|
||||
return doGetFluxChatResponse(inputRequest, observation)
|
||||
.doOnError(observation::error)
|
||||
.doFinally(s -> {
|
||||
observation.stop();
|
||||
})
|
||||
.contextWrite(ctx -> ctx.put(ObservationThreadLocalAccessor.KEY, observation));
|
||||
// @formatter:on
|
||||
});
|
||||
}
|
||||
|
||||
@@ -473,33 +477,93 @@ public class DefaultChatClient implements ChatClient {
|
||||
|
||||
var reqWithContext = new AdvisedRequestWithContext(toAdvisedRequest(inputRequest), advisorContext);
|
||||
|
||||
return Flux.fromIterable(inputRequest.advisors)
|
||||
return Flux.fromIterable(AdvisorObservableHelper.extractRequestAdvisors(inputRequest.advisors))
|
||||
.transformDeferredContextual((f, ctx) -> f
|
||||
// This allows us to call blocking code in reduce
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.reduce(reqWithContext, (rwc, advisor) -> {
|
||||
// Apply the Request advisors
|
||||
AdvisedRequest advisedRequest = AdvisorObservableHelper.adviseRequest(parentObservation,
|
||||
advisor, rwc.request, rwc.advisorContext);
|
||||
return new AdvisedRequestWithContext(advisedRequest, rwc.advisorContext);
|
||||
}))
|
||||
.single()
|
||||
.flatMapMany(rwc -> {
|
||||
DefaultChatClientRequestSpec advisedRequest = toDefaultChatClientRequestSpec(rwc.request,
|
||||
inputRequest.getObservationRegistry(), inputRequest.getCustomObservationConvention());
|
||||
|
||||
var prompt = toPrompt(advisedRequest, null);
|
||||
// Apply the around advisor chain that terminates with the, last,
|
||||
// model call advisor.
|
||||
Flux<ChatResponse> advisedResponse = inputRequest.aroundAdvisorChain.nextAroundStream(rwc.request,
|
||||
rwc.advisorContext);
|
||||
|
||||
Flux<ChatResponse> fluxChatResponse = this.chatModel.stream(prompt);
|
||||
|
||||
Flux<ChatResponse> advisedResponse = fluxChatResponse;
|
||||
// apply the advisors on response
|
||||
// Apply the Response advisors
|
||||
if (!CollectionUtils.isEmpty(inputRequest.getAdvisors())) {
|
||||
var currentAdvisors = new ArrayList<>(inputRequest.getAdvisors());
|
||||
for (RequestResponseAdvisor advisor : currentAdvisors) {
|
||||
advisedResponse = AdvisorObservableHelper.adviseResponse(parentObservation, advisor,
|
||||
advisedResponse, advisorContext);
|
||||
|
||||
var responseAdvisors = new ArrayList<>(
|
||||
AdvisorObservableHelper.extractResponseAdvisors(inputRequest.getAdvisors()));
|
||||
|
||||
List<ResponseAdvisor> perElementResponseAdvisors = responseAdvisors.stream()
|
||||
.filter(a -> a.getStreamResponseMode() == StreamResponseMode.PER_ELEMENT)
|
||||
.toList();
|
||||
|
||||
List<ResponseAdvisor> onFinishElementResponseAdvisors = responseAdvisors.stream()
|
||||
.filter(a -> a.getStreamResponseMode() == StreamResponseMode.ON_FINISH_ELEMENT)
|
||||
.toList();
|
||||
|
||||
// PER_ELEMENT and ON_FINISH_ELEMENT
|
||||
advisedResponse = advisedResponse.map(response -> {
|
||||
// PER_ELEMENT
|
||||
if (!CollectionUtils.isEmpty(perElementResponseAdvisors)) {
|
||||
for (ResponseAdvisor advisor : perElementResponseAdvisors) {
|
||||
response = AdvisorObservableHelper.adviseResponse(parentObservation, advisor,
|
||||
response, rwc.advisorContext);
|
||||
}
|
||||
}
|
||||
// ON_FINISH_ELEMENT
|
||||
if (!CollectionUtils.isEmpty(onFinishElementResponseAdvisors)) {
|
||||
for (ResponseAdvisor advisor : onFinishElementResponseAdvisors) {
|
||||
boolean withFinishReason = response.getResults()
|
||||
.stream()
|
||||
.filter(result -> result != null && result.getMetadata() != null
|
||||
&& StringUtils.hasText(result.getMetadata().getFinishReason()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
|
||||
if (withFinishReason) {
|
||||
response = AdvisorObservableHelper.adviseResponse(parentObservation, advisor,
|
||||
response, advisorContext);
|
||||
}
|
||||
}
|
||||
}
|
||||
return response;
|
||||
});
|
||||
|
||||
// CUSTOM
|
||||
// TODO: how to pass the parentObservation to the custom response
|
||||
// advisor?
|
||||
List<ResponseAdvisor> customResponseAdvisors = responseAdvisors.stream()
|
||||
.filter(a -> a.getStreamResponseMode() == StreamResponseMode.CUSTOM)
|
||||
.toList();
|
||||
if (!CollectionUtils.isEmpty(customResponseAdvisors)) {
|
||||
for (ResponseAdvisor advisor : customResponseAdvisors) {
|
||||
advisedResponse = advisor.adviseResponse(advisedResponse, rwc.advisorContext);
|
||||
}
|
||||
}
|
||||
|
||||
// AGGREGATE
|
||||
List<ResponseAdvisor> aggregateResponseAdvisors = responseAdvisors.stream()
|
||||
.filter(a -> a.getStreamResponseMode() == StreamResponseMode.AGGREGATE)
|
||||
.toList();
|
||||
|
||||
if (!CollectionUtils.isEmpty(aggregateResponseAdvisors)) {
|
||||
advisedResponse = new MessageAggregator().aggregate(advisedResponse, chatResponse -> {
|
||||
for (ResponseAdvisor advisor : aggregateResponseAdvisors) {
|
||||
AdvisorObservableHelper.adviseResponse(parentObservation, advisor, chatResponse,
|
||||
advisorContext);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return advisedResponse;
|
||||
|
||||
});
|
||||
@@ -547,52 +611,58 @@ public class DefaultChatClient implements ChatClient {
|
||||
|
||||
private final Map<String, Object> systemParams = new HashMap<>();
|
||||
|
||||
private final List<RequestResponseAdvisor> advisors = new ArrayList<>();
|
||||
private final List<Advisor> advisors = new ArrayList<>();
|
||||
|
||||
private final Map<String, Object> advisorParams = new HashMap<>();
|
||||
|
||||
private final DefaultAroundAdvisorChain aroundAdvisorChain;
|
||||
|
||||
public AroundAdvisorChain getAroundAdvisorChain() {
|
||||
return this.aroundAdvisorChain;
|
||||
}
|
||||
|
||||
private ObservationRegistry getObservationRegistry() {
|
||||
return observationRegistry;
|
||||
return this.observationRegistry;
|
||||
}
|
||||
|
||||
private ChatClientObservationConvention getCustomObservationConvention() {
|
||||
return customObservationConvention;
|
||||
return this.customObservationConvention;
|
||||
}
|
||||
|
||||
public String getUserText() {
|
||||
return userText;
|
||||
return this.userText;
|
||||
}
|
||||
|
||||
public Map<String, Object> getUserParams() {
|
||||
return userParams;
|
||||
return this.userParams;
|
||||
}
|
||||
|
||||
public String getSystemText() {
|
||||
return systemText;
|
||||
return this.systemText;
|
||||
}
|
||||
|
||||
public Map<String, Object> getSystemParams() {
|
||||
return systemParams;
|
||||
return this.systemParams;
|
||||
}
|
||||
|
||||
public ChatOptions getChatOptions() {
|
||||
return chatOptions;
|
||||
return this.chatOptions;
|
||||
}
|
||||
|
||||
public List<RequestResponseAdvisor> getAdvisors() {
|
||||
return advisors;
|
||||
public List<Advisor> getAdvisors() {
|
||||
return this.advisors;
|
||||
}
|
||||
|
||||
public Map<String, Object> getAdvisorParams() {
|
||||
return advisorParams;
|
||||
return this.advisorParams;
|
||||
}
|
||||
|
||||
public List<Message> getMessages() {
|
||||
return messages;
|
||||
return this.messages;
|
||||
}
|
||||
|
||||
public List<Media> getMedia() {
|
||||
return media;
|
||||
return this.media;
|
||||
}
|
||||
|
||||
public List<String> getFunctionNames() {
|
||||
@@ -600,7 +670,7 @@ public class DefaultChatClient implements ChatClient {
|
||||
}
|
||||
|
||||
public List<FunctionCallback> getFunctionCallbacks() {
|
||||
return functionCallbacks;
|
||||
return this.functionCallbacks;
|
||||
}
|
||||
|
||||
/* copy constructor */
|
||||
@@ -613,8 +683,8 @@ public class DefaultChatClient implements ChatClient {
|
||||
public DefaultChatClientRequestSpec(ChatModel chatModel, String userText, Map<String, Object> userParams,
|
||||
String systemText, Map<String, Object> systemParams, List<FunctionCallback> functionCallbacks,
|
||||
List<Message> messages, List<String> functionNames, List<Media> media, ChatOptions chatOptions,
|
||||
List<RequestResponseAdvisor> advisors, Map<String, Object> advisorParams,
|
||||
ObservationRegistry observationRegistry, ChatClientObservationConvention customObservationConvention) {
|
||||
List<Advisor> advisors, Map<String, Object> advisorParams, ObservationRegistry observationRegistry,
|
||||
ChatClientObservationConvention customObservationConvention) {
|
||||
|
||||
this.chatModel = chatModel;
|
||||
this.chatOptions = chatOptions != null ? chatOptions.copy()
|
||||
@@ -633,6 +703,40 @@ public class DefaultChatClient implements ChatClient {
|
||||
this.advisorParams.putAll(advisorParams);
|
||||
this.observationRegistry = observationRegistry;
|
||||
this.customObservationConvention = customObservationConvention;
|
||||
|
||||
// @formatter:off
|
||||
this.aroundAdvisorChain = DefaultAroundAdvisorChain.builder(observationRegistry)
|
||||
// At the stack bottom add the non-streaming and streaming model call advisors.
|
||||
// They play the role of the last advisor in the around advisor chain.
|
||||
.push(new CallAroundAdvisor() {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return CallAroundAdvisor.class.getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse aroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
|
||||
AroundAdvisorChain chain) {
|
||||
String formatParam = (String) adviceContext.get("formatParam");
|
||||
return chatModel.call(toPrompt(advisedRequest, formatParam));
|
||||
}
|
||||
})
|
||||
.push(new StreamAroundAdvisor() {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return StreamAroundAdvisor.class.getSimpleName();
|
||||
}
|
||||
@Override
|
||||
public Flux<ChatResponse> aroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
|
||||
AroundAdvisorChain chain) {
|
||||
return chatModel.stream(toPrompt(advisedRequest, null));
|
||||
}
|
||||
})
|
||||
.pushAll(this.advisors)
|
||||
.build();
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -662,18 +766,21 @@ public class DefaultChatClient implements ChatClient {
|
||||
consumer.accept(as);
|
||||
this.advisorParams.putAll(as.getParams());
|
||||
this.advisors.addAll(as.getAdvisors());
|
||||
this.aroundAdvisorChain.pushAll(as.getAdvisors());
|
||||
return this;
|
||||
}
|
||||
|
||||
public ChatClientRequestSpec advisors(RequestResponseAdvisor... advisors) {
|
||||
public ChatClientRequestSpec advisors(Advisor... advisors) {
|
||||
Assert.notNull(advisors, "the advisors must be non-null");
|
||||
this.advisors.addAll(Arrays.asList(advisors));
|
||||
this.aroundAdvisorChain.pushAll(Arrays.asList(advisors));
|
||||
return this;
|
||||
}
|
||||
|
||||
public ChatClientRequestSpec advisors(List<RequestResponseAdvisor> advisors) {
|
||||
public ChatClientRequestSpec advisors(List<Advisor> advisors) {
|
||||
Assert.notNull(advisors, "the advisors must be non-null");
|
||||
this.advisors.addAll(advisors);
|
||||
this.aroundAdvisorChain.pushAll(advisors);
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -797,11 +904,11 @@ public class DefaultChatClient implements ChatClient {
|
||||
}
|
||||
|
||||
public CallResponseSpec call() {
|
||||
return new DefaultCallResponseSpec(chatModel, this);
|
||||
return new DefaultCallResponseSpec(this);
|
||||
}
|
||||
|
||||
public StreamResponseSpec stream() {
|
||||
return new DefaultStreamResponseSpec(chatModel, this);
|
||||
return new DefaultStreamResponseSpec(this);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -906,4 +1013,4 @@ public class DefaultChatClient implements ChatClient {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import org.springframework.ai.chat.client.ChatClient.Builder;
|
||||
import org.springframework.ai.chat.client.ChatClient.PromptSystemSpec;
|
||||
import org.springframework.ai.chat.client.ChatClient.PromptUserSpec;
|
||||
import org.springframework.ai.chat.client.DefaultChatClient.DefaultChatClientRequestSpec;
|
||||
import org.springframework.ai.chat.client.advisor.api.Advisor;
|
||||
import org.springframework.ai.chat.client.observation.ChatClientObservationConvention;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||
@@ -69,7 +70,7 @@ public class DefaultChatClientBuilder implements Builder {
|
||||
return new DefaultChatClient(this.chatModel, this.defaultRequest);
|
||||
}
|
||||
|
||||
public Builder defaultAdvisors(RequestResponseAdvisor... advisor) {
|
||||
public Builder defaultAdvisors(Advisor... advisor) {
|
||||
this.defaultRequest.advisors(advisor);
|
||||
return this;
|
||||
}
|
||||
@@ -79,7 +80,7 @@ public class DefaultChatClientBuilder implements Builder {
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder defaultAdvisors(List<RequestResponseAdvisor> advisors) {
|
||||
public Builder defaultAdvisors(List<Advisor> advisors) {
|
||||
this.defaultRequest.advisors(advisors);
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -16,120 +16,41 @@
|
||||
|
||||
package org.springframework.ai.chat.client;
|
||||
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.MessageAggregator;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
/**
|
||||
* Advisor called before and after the {@link ChatModel#call(Prompt)} and
|
||||
* {@link ChatModel#stream(Prompt)} methods calls. The {@link ChatClient} maintains a
|
||||
* chain of advisors with chared execution context.
|
||||
* chain of advisors with shared advise context.
|
||||
*
|
||||
* @deprecated since 1.0.0 please use {@link RequestAdvisor}, {@link ResponseAdvisor}
|
||||
* instead.
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public interface RequestResponseAdvisor {
|
||||
@Deprecated
|
||||
public interface RequestResponseAdvisor extends RequestAdvisor, ResponseAdvisor {
|
||||
|
||||
public enum StreamResponseMode {
|
||||
|
||||
/**
|
||||
* The sync advisor will be called for each response chunk (e.g. on each Flux
|
||||
* item).
|
||||
*/
|
||||
PER_CHUNK,
|
||||
/**
|
||||
* The sync advisor is called only on chunks that contain a finish reason. Usually
|
||||
* the last chunk in the stream.
|
||||
*/
|
||||
ON_FINISH_REASON,
|
||||
/**
|
||||
* The sync advisor is called only once after the stream is completed and an
|
||||
* aggregated response is computed. Note that at that stage the advisor can not
|
||||
* modify the response, but only observe it and react on the aggregated response.
|
||||
*/
|
||||
AGGREGATE,
|
||||
/**
|
||||
* Delegates to the stream advisor implementation.
|
||||
*/
|
||||
CUSTOM;
|
||||
|
||||
}
|
||||
|
||||
default StreamResponseMode getStreamResponseMode() {
|
||||
return StreamResponseMode.CUSTOM;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the advisor name.
|
||||
*/
|
||||
@Override
|
||||
default String getName() {
|
||||
return this.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param request the {@link AdvisedRequest} data to be advised. Represents the row
|
||||
* {@link ChatClient.ChatClientRequestSpec} data before sealed into a {@link Prompt}.
|
||||
* @param context the shared data between the advisors in the chain. It is shared
|
||||
* between all request and response advising points of all advisors in the chain.
|
||||
* @return the advised {@link AdvisedRequest}.
|
||||
*/
|
||||
default AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
|
||||
@Override
|
||||
default AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> adviseContext) {
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param response the {@link ChatResponse} data to be advised. Represents the row
|
||||
* {@link ChatResponse} data after the {@link ChatModel#call(Prompt)} method is
|
||||
* called.
|
||||
* @param context the shared data between the advisors in the chain. It is shared
|
||||
* between all request and response advising points of all advisors in the chain.
|
||||
* @return the advised {@link ChatResponse}.
|
||||
*/
|
||||
default ChatResponse adviseResponse(ChatResponse response, Map<String, Object> context) {
|
||||
@Override
|
||||
default ChatResponse adviseResponse(ChatResponse response, Map<String, Object> adviseContext) {
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param fluxResponse the streaming {@link ChatResponse} data to be advised.
|
||||
* Represents the row {@link ChatResponse} stream data after the
|
||||
* {@link ChatModel#stream(Prompt)} method is called.
|
||||
* @param context the shared data between the advisors in the chain. It is shared
|
||||
* between all request and response advising points of all advisors in the chain.
|
||||
* @return the advised {@link ChatResponse} flux.
|
||||
*/
|
||||
default Flux<ChatResponse> adviseResponse(Flux<ChatResponse> fluxResponse, Map<String, Object> context) {
|
||||
|
||||
if (this.getStreamResponseMode() == StreamResponseMode.PER_CHUNK) {
|
||||
return fluxResponse.map(chatResponse -> this.adviseResponse(chatResponse, context));
|
||||
}
|
||||
else if (this.getStreamResponseMode() == StreamResponseMode.AGGREGATE) {
|
||||
return new MessageAggregator().aggregate(fluxResponse, chatResponse -> {
|
||||
this.adviseResponse(chatResponse, context);
|
||||
});
|
||||
}
|
||||
else if (this.getStreamResponseMode() == StreamResponseMode.ON_FINISH_REASON) {
|
||||
return fluxResponse.map(chatResponse -> {
|
||||
boolean withFinishReason = chatResponse.getResults()
|
||||
.stream()
|
||||
.filter(result -> result != null && result.getMetadata() != null
|
||||
&& StringUtils.hasText(result.getMetadata().getFinishReason()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
|
||||
if (withFinishReason) {
|
||||
return this.adviseResponse(chatResponse, context);
|
||||
}
|
||||
return chatResponse;
|
||||
});
|
||||
}
|
||||
|
||||
return fluxResponse;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,8 @@ package org.springframework.ai.chat.client.advisor;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.RequestResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
@@ -28,7 +29,7 @@ import org.springframework.util.Assert;
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0 M1
|
||||
*/
|
||||
public abstract class AbstractChatMemoryAdvisor<T> implements RequestResponseAdvisor {
|
||||
public abstract class AbstractChatMemoryAdvisor<T> implements RequestAdvisor, ResponseAdvisor {
|
||||
|
||||
public static final String CHAT_MEMORY_CONVERSATION_ID_KEY = "chat_memory_conversation_id";
|
||||
|
||||
@@ -60,8 +61,8 @@ public abstract class AbstractChatMemoryAdvisor<T> implements RequestResponseAdv
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamResponseMode getStreamResponseMode() {
|
||||
return StreamResponseMode.AGGREGATE;
|
||||
public String getName() {
|
||||
return this.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
protected T getChatMemoryStore() {
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package org.springframework.ai.chat.client.advisor;
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.advisor.api.Advisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
|
||||
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservableHelper;
|
||||
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservationContext;
|
||||
import org.springframework.ai.chat.client.advisor.observation.AdvisorObservationDocumentation;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import io.micrometer.observation.ObservationRegistry;
|
||||
import io.micrometer.observation.contextpropagation.ObservationThreadLocalAccessor;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public class DefaultAroundAdvisorChain implements AroundAdvisorChain {
|
||||
|
||||
private final Deque<CallAroundAdvisor> callAroundAdvisors;
|
||||
|
||||
private final Deque<StreamAroundAdvisor> streamAroundAdvisors;
|
||||
|
||||
private final ObservationRegistry observationRegistry;
|
||||
|
||||
public DefaultAroundAdvisorChain(ObservationRegistry observationRegistry) {
|
||||
this(observationRegistry, new ArrayDeque<CallAroundAdvisor>(), new ArrayDeque<StreamAroundAdvisor>());
|
||||
}
|
||||
|
||||
public DefaultAroundAdvisorChain(CallAroundAdvisor aroundAdvisor, ObservationRegistry observationRegistry) {
|
||||
this(observationRegistry, new ArrayDeque<CallAroundAdvisor>(), new ArrayDeque<StreamAroundAdvisor>());
|
||||
this.push(aroundAdvisor);
|
||||
}
|
||||
|
||||
public DefaultAroundAdvisorChain(ObservationRegistry observationRegistry,
|
||||
Deque<CallAroundAdvisor> callAroundAdvisors, Deque<StreamAroundAdvisor> streamAroundAdvisors) {
|
||||
Assert.notNull(callAroundAdvisors, "the callAroundAdvisors must be non-null");
|
||||
this.observationRegistry = observationRegistry;
|
||||
this.callAroundAdvisors = callAroundAdvisors;
|
||||
this.streamAroundAdvisors = streamAroundAdvisors;
|
||||
}
|
||||
|
||||
public DefaultAroundAdvisorChain(ObservationRegistry observationRegistry, List<Advisor> advisors) {
|
||||
this(observationRegistry);
|
||||
Assert.notNull(advisors, "the advisors must be non-null");
|
||||
advisors.forEach(this::push);
|
||||
}
|
||||
|
||||
public void pushAll(List<? extends Advisor> advisors) {
|
||||
Assert.notNull(advisors, "the advisors must be non-null");
|
||||
advisors.forEach(this::push);
|
||||
}
|
||||
|
||||
public void push(Advisor aroundAdvisor) {
|
||||
|
||||
Assert.notNull(aroundAdvisor, "the aroundAdvisor must be non-null");
|
||||
|
||||
if (aroundAdvisor instanceof CallAroundAdvisor callAroundAdvisor) {
|
||||
this.callAroundAdvisors.push(callAroundAdvisor);
|
||||
}
|
||||
// Note: the advisor can implement both the CallAroundAdvisor and
|
||||
// StreamAroundAdvisor.
|
||||
if (aroundAdvisor instanceof StreamAroundAdvisor streamAroundAdvisor) {
|
||||
this.streamAroundAdvisors.push(streamAroundAdvisor);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse nextAroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext) {
|
||||
|
||||
if (this.callAroundAdvisors.isEmpty()) {
|
||||
throw new IllegalStateException("No AroundAdvisor available to execute");
|
||||
}
|
||||
|
||||
var advisor = this.callAroundAdvisors.pop();
|
||||
|
||||
var observationContext = AdvisorObservationContext.builder()
|
||||
.withAdvisorName(advisor.getName())
|
||||
.withAdvisorType(AdvisorObservationContext.Type.AROUND)
|
||||
.withAdvisedRequest(advisedRequest)
|
||||
.withAdvisorRequestContext(adviceContext)
|
||||
.build();
|
||||
|
||||
return AdvisorObservationDocumentation.AI_ADVISOR
|
||||
.observation(null, AdvisorObservableHelper.DEFAULT_OBSERVATION_CONVENTION, () -> observationContext,
|
||||
this.observationRegistry)
|
||||
.observe(() -> advisor.aroundCall(advisedRequest, adviceContext, this));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<ChatResponse> nextAroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext) {
|
||||
|
||||
return Flux.deferContextual(contextView -> {
|
||||
|
||||
if (this.streamAroundAdvisors.isEmpty()) {
|
||||
return Flux.error(new IllegalStateException("No AroundAdvisor available to execute"));
|
||||
}
|
||||
|
||||
var advisor = this.streamAroundAdvisors.pop();
|
||||
|
||||
AdvisorObservationContext observationContext = AdvisorObservationContext.builder()
|
||||
.withAdvisorName(advisor.getName())
|
||||
.withAdvisorType(AdvisorObservationContext.Type.AROUND)
|
||||
.withAdvisedRequest(advisedRequest)
|
||||
.withAdvisorRequestContext(adviceContext)
|
||||
.build();
|
||||
|
||||
var observation = AdvisorObservationDocumentation.AI_ADVISOR.observation(null,
|
||||
AdvisorObservableHelper.DEFAULT_OBSERVATION_CONVENTION, () -> observationContext,
|
||||
this.observationRegistry);
|
||||
|
||||
observation.parentObservation(contextView.getOrDefault(ObservationThreadLocalAccessor.KEY, null)).start();
|
||||
|
||||
return advisor.aroundStream(advisedRequest, adviceContext, this)
|
||||
.doOnError(observation::error)
|
||||
.doFinally(s -> {
|
||||
observation.stop();
|
||||
})
|
||||
.contextWrite(ctx -> ctx.put(ObservationThreadLocalAccessor.KEY, observation));
|
||||
});
|
||||
}
|
||||
|
||||
public static Builder builder(ObservationRegistry observationRegistry) {
|
||||
return new Builder(observationRegistry);
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
|
||||
private final DefaultAroundAdvisorChain aroundAdvisorChain;
|
||||
|
||||
public Builder(ObservationRegistry observationRegistry) {
|
||||
this.aroundAdvisorChain = new DefaultAroundAdvisorChain(observationRegistry);
|
||||
}
|
||||
|
||||
public Builder push(Advisor aroundAdvisor) {
|
||||
Assert.notNull(aroundAdvisor, "the aroundAdvisor must be non-null");
|
||||
this.aroundAdvisorChain.push(aroundAdvisor);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder pushAll(List<Advisor> aroundAdvisors) {
|
||||
Assert.notNull(aroundAdvisors, "the aroundAdvisors must be non-null");
|
||||
this.aroundAdvisorChain.pushAll(aroundAdvisors);
|
||||
return this;
|
||||
}
|
||||
|
||||
public DefaultAroundAdvisorChain build() {
|
||||
return this.aroundAdvisorChain;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -22,7 +22,8 @@ import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.RequestResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.model.Content;
|
||||
@@ -40,7 +41,7 @@ import org.springframework.util.StringUtils;
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public class QuestionAnswerAdvisor implements RequestResponseAdvisor {
|
||||
public class QuestionAnswerAdvisor implements RequestAdvisor, ResponseAdvisor {
|
||||
|
||||
private static final String DEFAULT_USER_TEXT_ADVISE = """
|
||||
Context information is below.
|
||||
@@ -91,6 +92,11 @@ public class QuestionAnswerAdvisor implements RequestResponseAdvisor {
|
||||
this.userTextAdvise = userTextAdvise;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return this.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
|
||||
|
||||
@@ -140,9 +146,4 @@ public class QuestionAnswerAdvisor implements RequestResponseAdvisor {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamResponseMode getStreamResponseMode() {
|
||||
return StreamResponseMode.ON_FINISH_REASON;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,8 @@ import java.util.function.Function;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.RequestResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.model.ModelOptionsUtils;
|
||||
|
||||
@@ -30,7 +31,7 @@ import org.springframework.ai.model.ModelOptionsUtils;
|
||||
*
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
public class SimpleLoggerAdvisor implements RequestResponseAdvisor {
|
||||
public class SimpleLoggerAdvisor implements RequestAdvisor, ResponseAdvisor {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SimpleLoggerAdvisor.class);
|
||||
|
||||
@@ -56,6 +57,11 @@ public class SimpleLoggerAdvisor implements RequestResponseAdvisor {
|
||||
this.responseToString = responseToString;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return this.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
|
||||
logger.debug("request: {}", this.requestToString.apply(request));
|
||||
@@ -73,9 +79,4 @@ public class SimpleLoggerAdvisor implements RequestResponseAdvisor {
|
||||
return SimpleLoggerAdvisor.class.getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamResponseMode getStreamResponseMode() {
|
||||
return StreamResponseMode.AGGREGATE;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright 2024 - 2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.ai.chat.client.advisor.api;
|
||||
|
||||
import org.springframework.ai.chat.client.advisor.DefaultAroundAdvisorChain;
|
||||
|
||||
/**
|
||||
* Parent advisor interface for all advisors.
|
||||
*
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
* @see {@link RequestAdvisor}, {@link ResponseAdvisor}, {@link CallAroundAdvisor},
|
||||
* {@link StreamAroundAdvisor}, {@link DefaultAroundAdvisorChain}
|
||||
*/
|
||||
public interface Advisor {
|
||||
|
||||
/**
|
||||
* @return the advisor name.
|
||||
*/
|
||||
String getName();
|
||||
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package org.springframework.ai.chat.client.advisor.api;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public interface AroundAdvisorChain {
|
||||
|
||||
ChatResponse nextAroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext);
|
||||
|
||||
Flux<ChatResponse> nextAroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2024 - 2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.ai.chat.client.advisor.api;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
*/
|
||||
|
||||
public interface CallAroundAdvisor extends Advisor {
|
||||
|
||||
/**
|
||||
* Around advice that wraps the {@link ChatModel#call(Prompt)} method.
|
||||
* @param advisedRequest the advised request
|
||||
* @param adviceContext the advice context
|
||||
* @param chain the advisor chain
|
||||
* @return the response
|
||||
*/
|
||||
ChatResponse aroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext, AroundAdvisorChain chain);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright 2024-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.chat.client.advisor.api;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
/**
|
||||
* Advisor called before the {@link ChatModel#call(Prompt)} and
|
||||
* {@link ChatModel#stream(Prompt)} methods are called. The {@link ChatClient} maintains a
|
||||
* chain of advisors with shared advise context.
|
||||
*
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public interface RequestAdvisor extends Advisor {
|
||||
|
||||
/**
|
||||
* @param request the {@link AdvisedRequest} data to be advised. Represents the row
|
||||
* {@link ChatClient.ChatClientRequestSpec} data before sealed into a {@link Prompt}.
|
||||
* @param adviseContext the shared data between the advisors in the chain. It is
|
||||
* shared between all request and response advising points of all advisors in the
|
||||
* chain.
|
||||
* @return the advised {@link AdvisedRequest}.
|
||||
*/
|
||||
AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> adviseContext);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/*
|
||||
* Copyright 2024-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.chat.client.advisor.api;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
/**
|
||||
* Advisor called after the {@link ChatModel#call(Prompt)} (or
|
||||
* {@link ChatModel#stream(Prompt)}) method call. The {@link ChatClient} maintains a chain
|
||||
* of advisors with shared advise context.
|
||||
*
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public interface ResponseAdvisor extends Advisor {
|
||||
|
||||
/**
|
||||
* @param response the {@link ChatResponse} data to be advised. Represents the row
|
||||
* {@link ChatResponse} data after the {@link ChatModel#call(Prompt)} method is
|
||||
* called.
|
||||
* @param adviseContext the shared data between the advisors in the chain. It is
|
||||
* shared between all request and response advising points of all advisors in the
|
||||
* chain.
|
||||
* @return the advised {@link ChatResponse}.
|
||||
*/
|
||||
ChatResponse adviseResponse(ChatResponse response, Map<String, Object> adviseContext);
|
||||
|
||||
/**
|
||||
* Different modes of advising the streaming responses.
|
||||
*/
|
||||
public enum StreamResponseMode {
|
||||
|
||||
/**
|
||||
* Called for each response element in the Flux. The response advisor can modify
|
||||
* the elements before they are returned to the client.
|
||||
*/
|
||||
PER_ELEMENT,
|
||||
/**
|
||||
* Called only on Flux elements that contain a finish reason. Usually the last
|
||||
* element in the Flux. The response advisor can modify the elements before they
|
||||
* are returned to the client.
|
||||
*/
|
||||
ON_FINISH_ELEMENT,
|
||||
/**
|
||||
* Called only once after all Flux elements have been consumed. All elements are
|
||||
* merged into a single ChatResponse element and provided to the response advisor
|
||||
* to process. <br/>
|
||||
* Mind that at that stage the response advisor can not longer modify the response
|
||||
* returned to the client.
|
||||
*/
|
||||
AGGREGATE,
|
||||
/**
|
||||
* Delegates to the stream advisor implementation.
|
||||
*/
|
||||
CUSTOM;
|
||||
|
||||
}
|
||||
|
||||
default StreamResponseMode getStreamResponseMode() {
|
||||
return StreamResponseMode.AGGREGATE;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param fluxResponse the streaming {@link ChatResponse} data to be advised.
|
||||
* Represents the row {@link ChatResponse} stream data after the
|
||||
* {@link ChatModel#stream(Prompt)} method is called.
|
||||
* @param adviseContext the shared data between the advisors in the chain. It is
|
||||
* shared between all request and response advising points of all advisors in the
|
||||
* chain.
|
||||
* @return the advised {@link ChatResponse} flux.
|
||||
*/
|
||||
default Flux<ChatResponse> adviseResponse(Flux<ChatResponse> fluxResponse, Map<String, Object> adviseContext) {
|
||||
return fluxResponse;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2024 - 2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.ai.chat.client.advisor.api;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public interface StreamAroundAdvisor extends Advisor {
|
||||
|
||||
/**
|
||||
* Around advice that wraps the invocation of the advised request.
|
||||
* @param advisedRequest
|
||||
* @param adviceContext
|
||||
* @param chain the chain of advisors to execute
|
||||
* @return the result of the advised request
|
||||
*/
|
||||
Flux<ChatResponse> aroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
|
||||
AroundAdvisorChain chain);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* Copyright 2024 - 2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.ai.chat.client.advisor.around;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
|
||||
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.MessageType;
|
||||
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.model.MessageAggregator;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.vectorstore.SearchRequest;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public class CacheAroundAdvisor implements CallAroundAdvisor, StreamAroundAdvisor {
|
||||
|
||||
private final VectorStore vectorStore;
|
||||
|
||||
private static final String DOCUMENT_METADATA_ADVISOR_CACHE_TAG = "advisorCacheDocument";
|
||||
|
||||
private static final String DOCUMENT_METADATA_ADVISOR_CACHE_RESPONSE = "advisorCacheResponse";
|
||||
|
||||
public CacheAroundAdvisor(VectorStore vectorStore) {
|
||||
this.vectorStore = vectorStore;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return this.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse aroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
|
||||
AroundAdvisorChain chain) {
|
||||
|
||||
var cachedResponseOption = getCacheEntry(advisedRequest, adviceContext);
|
||||
if (cachedResponseOption.isPresent()) {
|
||||
return cachedResponseOption.get();
|
||||
}
|
||||
|
||||
ChatResponse chatResponse = chain.nextAroundCall(advisedRequest, adviceContext);
|
||||
|
||||
saveCacheEntry(advisedRequest.userText(), chatResponse);
|
||||
|
||||
return chatResponse;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<ChatResponse> aroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
|
||||
AroundAdvisorChain chain) {
|
||||
|
||||
var cachedResponseOption = getCacheEntry(advisedRequest, adviceContext);
|
||||
if (cachedResponseOption.isPresent()) {
|
||||
return Flux.just(cachedResponseOption.get());
|
||||
}
|
||||
|
||||
Flux<ChatResponse> fluxChatResponse = chain.nextAroundStream(advisedRequest, adviceContext);
|
||||
|
||||
return new MessageAggregator().aggregate(fluxChatResponse, chatResponse -> {
|
||||
saveCacheEntry(advisedRequest.userText(), chatResponse);
|
||||
});
|
||||
}
|
||||
|
||||
private void saveCacheEntry(String userQuestion, ChatResponse chatResponse) {
|
||||
List<Message> assistantMessages = chatResponse.getResults().stream().map(g -> (Message) g.getOutput()).toList();
|
||||
if (!CollectionUtils.isEmpty(assistantMessages)) {
|
||||
this.vectorStore.add(toDocuments(userQuestion, assistantMessages));
|
||||
}
|
||||
}
|
||||
|
||||
private Optional<ChatResponse> getCacheEntry(AdvisedRequest advisedRequest, Map<String, Object> adviceContext) {
|
||||
|
||||
// TODO: convert into pompty first or at least materialize the user params.
|
||||
String userText = advisedRequest.userText();
|
||||
|
||||
// @formatter:off
|
||||
var searchRequest = SearchRequest.query(userText)
|
||||
.withSimilarityThreshold(0.95)
|
||||
.withTopK(1)
|
||||
.withFilterExpression("'"+ DOCUMENT_METADATA_ADVISOR_CACHE_TAG + "' == 'true'");
|
||||
// @formatter:on
|
||||
|
||||
List<Document> doc = vectorStore.similaritySearch(searchRequest);
|
||||
|
||||
// return cached response
|
||||
return CollectionUtils.isEmpty(doc) ? Optional.empty() : Optional.of(fromDocument(doc.get(0)));
|
||||
}
|
||||
|
||||
private ChatResponse fromDocument(Document doc) {
|
||||
|
||||
if (!doc.getMetadata().containsKey(DOCUMENT_METADATA_ADVISOR_CACHE_RESPONSE)) {
|
||||
throw new IllegalStateException("The document is missing the cache response metadata!");
|
||||
}
|
||||
String cachedResponse = "" + doc.getMetadata().get(DOCUMENT_METADATA_ADVISOR_CACHE_RESPONSE);
|
||||
|
||||
return ChatResponse.builder()
|
||||
.withGenerations(List.of(new Generation(new AssistantMessage(cachedResponse, Map.of()),
|
||||
ChatGenerationMetadata.from("STOP", null))))
|
||||
.build();
|
||||
}
|
||||
|
||||
private List<Document> toDocuments(String userQuestion, List<Message> messages) {
|
||||
|
||||
List<Document> docs = messages.stream()
|
||||
.filter(m -> m.getMessageType() == MessageType.ASSISTANT)
|
||||
.map(message -> {
|
||||
var metadata = new HashMap<>(message.getMetadata() != null ? message.getMetadata() : new HashMap<>());
|
||||
metadata.put(DOCUMENT_METADATA_ADVISOR_CACHE_TAG, "true");
|
||||
metadata.put(DOCUMENT_METADATA_ADVISOR_CACHE_RESPONSE, message.getContent());
|
||||
// TODO: Pehaps we need to serialize the message metadata to the document
|
||||
|
||||
return new Document(userQuestion, metadata);
|
||||
|
||||
})
|
||||
.toList();
|
||||
|
||||
return docs;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package org.springframework.ai.chat.client.advisor.around;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.advisor.api.AroundAdvisorChain;
|
||||
import org.springframework.ai.chat.client.advisor.api.CallAroundAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.StreamAroundAdvisor;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
/**
|
||||
* A {@link CallAroundAdvisor} and {@link StreamAroundAdvisor} that filters out the
|
||||
* response if the user input contains any of the sensitive words.
|
||||
*
|
||||
* @author Christian Tzolov
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public class SafeGuardAroundAdvisor implements CallAroundAdvisor, StreamAroundAdvisor {
|
||||
|
||||
private final List<String> sensitiveWords;
|
||||
|
||||
public SafeGuardAroundAdvisor(List<String> sensitiveWords) {
|
||||
this.sensitiveWords = sensitiveWords;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return this.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse aroundCall(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
|
||||
AroundAdvisorChain chain) {
|
||||
|
||||
if (!CollectionUtils.isEmpty(this.sensitiveWords)
|
||||
&& sensitiveWords.stream().anyMatch(w -> advisedRequest.userText().contains(w))) {
|
||||
return ChatResponse.builder().withGenerations(List.of()).build();
|
||||
}
|
||||
|
||||
return chain.nextAroundCall(advisedRequest, adviceContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<ChatResponse> aroundStream(AdvisedRequest advisedRequest, Map<String, Object> adviceContext,
|
||||
AroundAdvisorChain chain) {
|
||||
|
||||
if (!CollectionUtils.isEmpty(this.sensitiveWords)
|
||||
&& sensitiveWords.stream().anyMatch(w -> advisedRequest.userText().contains(w))) {
|
||||
return Flux.empty();
|
||||
}
|
||||
|
||||
return chain.nextAroundStream(advisedRequest, adviceContext);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -15,17 +15,19 @@
|
||||
*/
|
||||
package org.springframework.ai.chat.client.advisor.observation;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.RequestResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.RequestResponseAdvisor.StreamResponseMode;
|
||||
import org.springframework.ai.chat.client.advisor.api.Advisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.RequestAdvisor;
|
||||
import org.springframework.ai.chat.client.advisor.api.ResponseAdvisor;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.MessageAggregator;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import io.micrometer.observation.Observation;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
@@ -33,9 +35,9 @@ import reactor.core.publisher.Flux;
|
||||
*/
|
||||
public abstract class AdvisorObservableHelper {
|
||||
|
||||
private static final AdvisorObservationConvention DEFAULT_OBSERVATION_CONVENTION = new DefaultAdvisorObservationConvention();
|
||||
public static final AdvisorObservationConvention DEFAULT_OBSERVATION_CONVENTION = new DefaultAdvisorObservationConvention();
|
||||
|
||||
public static AdvisedRequest adviseRequest(Observation parentObservation, RequestResponseAdvisor advisor,
|
||||
public static AdvisedRequest adviseRequest(Observation parentObservation, RequestAdvisor advisor,
|
||||
AdvisedRequest advisedRequest, Map<String, Object> advisorContext) {
|
||||
|
||||
var observationContext = AdvisorObservationContext.builder()
|
||||
@@ -52,7 +54,7 @@ public abstract class AdvisorObservableHelper {
|
||||
.observe(() -> advisor.adviseRequest(advisedRequest, advisorContext));
|
||||
}
|
||||
|
||||
public static ChatResponse adviseResponse(Observation parentObservation, RequestResponseAdvisor advisor,
|
||||
public static ChatResponse adviseResponse(Observation parentObservation, ResponseAdvisor advisor,
|
||||
ChatResponse response, Map<String, Object> advisorContext) {
|
||||
|
||||
var observationContext = AdvisorObservationContext.builder()
|
||||
@@ -68,35 +70,34 @@ public abstract class AdvisorObservableHelper {
|
||||
.observe(() -> advisor.adviseResponse(response, advisorContext));
|
||||
}
|
||||
|
||||
public static Flux<ChatResponse> adviseResponse(Observation parentObservation, RequestResponseAdvisor advisor,
|
||||
Flux<ChatResponse> fluxResponse, Map<String, Object> advisorContext) {
|
||||
public static List<RequestAdvisor> extractRequestAdvisors(List<Advisor> advisors) {
|
||||
return advisors.stream()
|
||||
.filter(advisor -> advisor instanceof RequestAdvisor)
|
||||
.map(a -> (RequestAdvisor) a)
|
||||
.toList();
|
||||
}
|
||||
|
||||
if (advisor.getStreamResponseMode() == StreamResponseMode.PER_CHUNK) {
|
||||
return fluxResponse
|
||||
.map(chatResponse -> adviseResponse(parentObservation, advisor, chatResponse, advisorContext));
|
||||
}
|
||||
else if (advisor.getStreamResponseMode() == StreamResponseMode.AGGREGATE) {
|
||||
return new MessageAggregator().aggregate(fluxResponse, chatResponse -> {
|
||||
adviseResponse(parentObservation, advisor, chatResponse, advisorContext);
|
||||
});
|
||||
}
|
||||
else if (advisor.getStreamResponseMode() == StreamResponseMode.ON_FINISH_REASON) {
|
||||
return fluxResponse.map(chatResponse -> {
|
||||
boolean withFinishReason = chatResponse.getResults()
|
||||
.stream()
|
||||
.filter(result -> result != null && result.getMetadata() != null
|
||||
&& StringUtils.hasText(result.getMetadata().getFinishReason()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
/**
|
||||
* Extracts the {@link ResponseAdvisor} instances from the given list of advisors and
|
||||
* returns them in reverse order.
|
||||
* @param advisors list of all registered advisor types.
|
||||
* @return the list of {@link ResponseAdvisor} instances in reverse order.
|
||||
*/
|
||||
public static List<ResponseAdvisor> extractResponseAdvisors(List<Advisor> advisors) {
|
||||
|
||||
if (withFinishReason) {
|
||||
return adviseResponse(parentObservation, advisor, chatResponse, advisorContext);
|
||||
}
|
||||
return chatResponse;
|
||||
});
|
||||
var list = advisors.stream()
|
||||
.filter(advisor -> advisor instanceof ResponseAdvisor)
|
||||
.map(a -> (ResponseAdvisor) a)
|
||||
.toList();
|
||||
|
||||
// reverse the list
|
||||
if (CollectionUtils.isEmpty(list)) {
|
||||
return list;
|
||||
}
|
||||
|
||||
return advisor.adviseResponse(fluxResponse, advisorContext);
|
||||
var reversedList = new ArrayList<>(list);
|
||||
Collections.reverse(reversedList);
|
||||
return Collections.unmodifiableList(reversedList);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,12 +25,14 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.ai.chat.client.AdvisedRequest;
|
||||
import org.springframework.ai.chat.client.DefaultChatClient.DefaultChatClientRequestSpec;
|
||||
import org.springframework.ai.chat.client.RequestResponseAdvisor;
|
||||
import org.springframework.ai.chat.client.observation.ChatClientObservationDocumentation.HighCardinalityKeyNames;
|
||||
import org.springframework.ai.chat.client.observation.ChatClientObservationDocumentation.LowCardinalityKeyNames;
|
||||
import org.springframework.ai.chat.metadata.Usage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.model.function.FunctionCallback;
|
||||
|
||||
import io.micrometer.common.KeyValue;
|
||||
@@ -93,6 +95,17 @@ class DefaultChatClientObservationConventionTests {
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdvisedRequest adviseRequest(AdvisedRequest request, Map<String, Object> context) {
|
||||
return request;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse adviseResponse(ChatResponse response, Map<String, Object> adviseContext) {
|
||||
return response;
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user