diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/client/advisor/VectorStoreChatMemoryAdvisor.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/client/advisor/VectorStoreChatMemoryAdvisor.java index 6b042b176..bb3c24772 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/client/advisor/VectorStoreChatMemoryAdvisor.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/client/advisor/VectorStoreChatMemoryAdvisor.java @@ -68,6 +68,11 @@ public class VectorStoreChatMemoryAdvisor extends AbstractChatMemoryAdvisor output = chatClient.prompt() .user("Tell me a joke") .stream() @@ -132,16 +134,278 @@ Flux output = chatClient.prompt() You can also stream the `ChatResponse` using the method `Flux chatResponse()`. -TIP: The chunked stream response can not be converted into a Java entities automatically. -Use the xref:api/structured-output-converter.adoc#_structuredoutputconverter[Structured Output Converter] to convert the aggregated response. +In the 1.0.0 M2 we will offer a convenience method that will let you return an Java entity with the reactive `stream()` method. +In the meantime, you should use the xref:api/structured-output-converter.adoc#StructuredOutputConverter[Structured Output Converter] to convert the aggregated response explicity as shown below. +This also demonstrates the use of parameters in the fluent API that will be discussed in more detail in a later section of the documentation. -== Using defaults and parameters +[source,java] +---- + var converter = new BeanOutputConverter<>(new ParameterizedTypeReference>() { + }); -It is often useful to create a `ChatClient` with default user and/or system text defined at design time. -By design time, we mean creating it at application startup in a `@Configuration` class. -You can then replace the user or system parameters in the main runtime code. + Flux flux = this.chatClient.prompt() + .user(u -> u.text(""" + Generate the filmography for a random actor. + {format} + """) + .param("format", converter.getFormat())) + .stream() + .content(); -Here is a simple example which defines the system text without using any parameters. -The system text sets the context, instructions, and overall desired behavior for the model's response + String content = flux.collectList().block().stream().collect(Collectors.joining()); -== Passing in user and system parameters + List actorFilms = converter.convert(content); +---- + +== call() return values + +After specifying the `call` method on `ChatClient` there are a few different options for the response type. + +* `String content()`: returns the String content of the response +* `ChatResponse chatResponse()`: returns the `ChatResponse` object that contains multiple generations and also metadata about the response, for example how many token were used to create the response. +* `entity` to return a Java type +** entity(ParameterizedTypeReference type): used to return a Collection of entity types. +** entity(Class type): used to return a specific entity type. +** entity(StructuredOutputConverter structuredOutputConverter): used to specify an instance of a `StructuredOutputConverter` to convert a `String` to an entity type. + +You can also invoke the `stream` method instead of `call` and + + +== stream() return values + +After specifying the `stream` method on `ChatClient`, there are a few options for the response type: + +* `Flux content()`: Returns a Flux of the string being generated by the AI model. +* `Flux chatResponse()`: Returns a Flux of the `ChatResponse` object, which contains additional metadata about the response. + +== Using Defaults + +Creating a ChatClient with default system text in an `@Configuration` class simplifies runtime code. +By setting defaults, you only need to specify user text when calling `ChatClient, eliminating the need to set system text for each request in your runtime codeala path. + + + +=== Default System Text + +In the following example, we will configure the system text to always reply in a pirate's voice. +To avoid repeating the system text in runtime code, we will create a `ChatClient` instance in an `@Configuration` class. +[source,java] +---- +@Configuration +class Config { + + @Bean + ChatClient chatClient(ChatClient.Builder builder) { + return builder.defaultSystem("You are a friendly chat bot that answers question in the voice of a Pirate") + .build(); + } + +} +---- + +and an `@RestController` to invoke it + +[source,java] +---- +@RestController +class AIController { + + private final ChatClient chatClient; + + AIController(ChatClient chatClient) { + this.chatClient = chatClient; + } + + @GetMapping("/ai/simple") + public Map completion(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) { + return Map.of("completion", chatClient.prompt().user(message).call().content()); + } +} +---- + +invoking it via curl gives + +[source,bash] +---- +❯ curl localhost:8080/ai/simple +{"generation":"Why did the pirate go to the comedy club? To hear some arrr-rated jokes! Arrr, matey!"} +---- + +=== Default System Text with parameters + +In the following example, we will use a placeholder in the system text to specify the voice of the completion at runtime instead of design time. + +[source,java] +---- +@Configuration +class Config { + + @Bean + ChatClient chatClient(ChatClient.Builder builder) { + return builder.defaultSystem("You are a friendly chat bot that answers question in the voice of a {voice}") + .build(); + } + +} +---- + +[source,java] +---- +@RestController +class AIController { + private final ChatClient chatClient + AIController(ChatClient chatClient) { + this.chatClient = chatClient; + } + @GetMapping("/ai") + Map completion(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message, String voice) { + return Map.of( + "completion", + chatClient.prompt() + .system(sp -> sp.param("voice", voice)) + .user(message) + .call() + .content()); + } +} +---- + +The response is + +[source.bash] +---- +http localhost:8080/ai voice=='Robert DeNiro' +{ + "completion": "You talkin' to me? Okay, here's a joke for ya: Why couldn't the bicycle stand up by itself? Because it was two tired! Classic, right?" +} +---- + +=== Other defaults + +At the `ChatClient.Builder` level, you can specify the default prompt. + +* `defaultOptions(ChatOptions chatOptions)`: Pass in either portable options defined in the `ChatOptions` class or model-specific options such as those in `OpenAiChatOptions`. For more information on model-specific `ChatOptions` implementations, refer to the JavaDocs. + +* `defaultFunction(String name, String description, java.util.function.Function function)`: The `name` is used to refer to the function in user text. The `description` explains the function's purpose and helps the AI model choose the correct function for an accurate response. The `function` argument is a Java function instance that the model will execute when necessary. + +* `defaultFunctions(String... functionNames)`: The bean names of `java.util.Function`s defined in the application context. + +* `defaultUser(String text)`, `defaultUser(Resource text)`, `defaultUser(Consumer userSpecConsumer)`: These methods let you define the user text. The `Consumer` allows you to use a lambda to specify the user text and any default parameters. + + +* `defaultAdvisors(RequestResponseAdvisor... advisor)`: Advisors allow modification of the data used to create the `Prompt`. The `QuestionAnswerAdvisor` implementation enables the pattern of `Retrieval Augmented Generation` by appending the prompt with context information related to the user text. + +* `defaultAdvisors(Consumer advisorSpecConsumer)`: This method allows you to define a `Consumer` to configure multiple advisors using the `AdvisorSpec`. Advisors can modify the data used to create the final `Prompt`. The `Consumer` lets you specify a lambda to add advisors, such as `QuestionAnswerAdvisor`, which supports `Retrieval Augmented Generation` by appending the prompt with relevant context information based on the user text. + +You can override these defaults at runtime using the corresponding methods without the `default` prefix. + +* `options(ChatOptions chatOptions)` + +* `function(String name, String description, +java.util.function.Function function)` + +* `functions(String... functionNames) + +* `user(String text)` , `user(Resource text)`, `user(Consumer userSpecConsumer)` + +* `advisors(RequestResponseAdvisor... advisor)` + +* `advisors(Consumer advisorSpecConsumer)` + + +== Advisors + +A common pattern when calling an AI model with user text is to append or augment the prompt with contextual data. + +This contextual data can be of different types. Common types include: + +* **Your own data**: This is data the AI model hasn't been trained on. Even if the model has seen similar data, the appended contextual data takes precedence in generating the response. + +* **Conversational history**: The chat model's API is stateless. If you tell the AI model your name, it won't remember it in subsequent interactions. Conversational history must be sent with each request to ensure previous interactions are considered when generating a response. + +=== Retrieval Augmented Generation + +A vector database stores data that the AI model is unaware of. +When a user question is sent to the AI model, a `QuestionAnswerAdvisor` queries the vector database for documents related to the user question. + +The response from the vector database is appended to the user text to provide context for the AI model to generate a response. + +Assuming you have already loaded data into a `VectorStore`, you can perform Retrieval Augmented Generation (RAG) by providing an instance of `QuestionAnswerAdvisor` to the `ChatClient`. + + +[source,java] +---- +ChatResponse response = ChatClient.builder(chatModel) + .build().prompt() + .advisors(new QuestionAnswerAdvisor(vectorStore, SearchRequest.defaults())) + .user(userText) + .call() + .chatResponse(); +---- + +Is this example, the `SearchRequest.defaults()` will perform a similarity search over all documents in the Vector Database. +To restrict the types of documents that are searched, the `SearchRequest` takes a SQL like filter expression that is portable across all `VectorStores`. + +=== Chat Memory + +The interface `ChatMemory` represents a storage for chat conversation history. It provides methods to add messages to a +* conversation, retrieve messages from a conversation, and clear the conversation history. + +There is one implementation `InMemoryChatMemory` that provides in-memory storage for chat conversation history. + +Two advisor implementations use the `ChatMemory` interface to advice the prompt with conversation history which differ in the details of how the memory is added to the prompt + +* `MessageChatMemoryAdvisor` : Memory is retrieved added as a collection of messages to the prompt +* `PromptChatMemoryAdvisor` : Memory is retrieved added into the prompt's system text. + + +* `VectorStoreChatMemoryAdvisor` : The construtor ` VectorStoreChatMemoryAdvisor(VectorStore vectorStore, String defaultConversationId, int chatHistoryWindowSize)` lets you specify the VectorStore to retrieve the chat history from, the unqiue conversation ID, the size of the chat history to be retreived in token size. + +A sample `@Service` implementaiton that uses several advisors is shown below + +[source,java] +---- + +import static org.springframework.ai.chat.client.advisor.AbstractChatMemoryAdvisor.CHAT_MEMORY_CONVERSATION_ID_KEY; +import static org.springframework.ai.chat.client.advisor.AbstractChatMemoryAdvisor.CHAT_MEMORY_RETRIEVE_SIZE_KEY; + +@Service +public class CustomerSupportAssistant { + + private final ChatClient chatClient; + + public CustomerSupportAssistant(ChatClient.Builder builder, VectorStore vectorStore, ChatMemory chatMemory) { + + this.chatClient = builderh + .defaultSystem(""" + You are a customer chat support agent of an airline named "Funnair".", Respond in a friendly, + helpful, and joyful manner. + + Before providing information about a booking or cancelling a booking, you MUST always + get the following information from the user: booking number, customer first name and last name. + + Before changing a booking you MUST ensure it is permitted by the terms. + + If there is a charge for the change, you MUST ask the user to consent before proceeding. + """) + .defaultAdvisors( + new PromptChatMemoryAdvisor(chatMemory), + // new MessageChatMemoryAdvisor(chatMemory), // CHAT MEMORY + new QuestionAnswerAdvisor(vectorStore, SearchRequest.defaults()), + new LoggingAdvisor()) // RAG + .defaultFunctions("getBookingDetails", "changeBooking", "cancelBooking") // FUNCTION CALLING + .build(); +} + +public Flux chat(String chatId, String userMessageContent) { + + return this.chatClient.prompt() + .user(userMessageContent) + .advisors(a -> a + .param(CHAT_MEMORY_CONVERSATION_ID_KEY, chatId) + .param(CHAT_MEMORY_RETRIEVE_SIZE_KEY, 100)) + .stream().content(); + } +} + +---- diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/testing.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/testing.adoc index 8ea3c637d..9836a56d1 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/testing.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/testing.adoc @@ -1,3 +1,95 @@ = Evaluation Testing -TBD \ No newline at end of file +Testing AI applications requires evaluating the generated content to ensure the AI model has not produced a hallucinated response. + +One method to evaluate the response is to use the AI model itself for evaluation. Select the best AI model for the evaluation, which may not be the same model used to generate the response. + +The Spring AI interface for evaluating responses is `Evaluator`, defined as: + + + +[source,java] +---- +@FunctionalInterface +public interface Evaluator { + EvaluationResponse evaluate(EvaluationRequest evaluationRequest) +} +---- + +The input to the evaluation is the `EvaluationRequest` defined as + +[source,java] +---- +public class EvaluationRequest { + + private final String userText; + + private final List dataList; + + private final ChatResponse chatResponse; + + public EvaluationRequest(String userText, List dataList, ChatResponse chatResponse) { + this.userText = userText; + this.dataList = dataList; + this.chatResponse = chatResponse; + } + + ... +} +---- + +* `userText`: The raw input from the user. +* `dataList`: Contextual data, such as from Retrieval Augmented Generation, appended to the raw input. +* `chatResponse`: The AI model's response. + +== RelevancyEvaluator + +One implementation is the `RelevancyEvaluator`, which uses the AI model for evaluation. More implementations will be available in future releases. + +The `RelevancyEvaluator` uses the input (`userText`) and the AI model's output (`chatResponse`) to ask the question: + +[source, text] +---- +Your task is to evaluate if the response for the query +is in line with the context information provided.\n +You have two options to answer. Either YES/ NO.\n +Answer - YES, if the response for the query +is in line with context information otherwise NO.\n +Query: \n {query}\n +Response: \n {response}\n +Context: \n {context}\n +Answer: " +---- + +Here is an example of a JUnit test that performs a RAG query over a PDF document loaded into a Vector Store and then evaluates if the response is relevant to the user text. + +[source,java] +---- +@Test +void testEvaluation() { + + dataController.delete(); + dataController.load(); + + String userText = "What is the purpose of Carina?"; + + ChatResponse response = ChatClient.builder(chatModel) + .build().prompt() + .advisors(new QuestionAnswerAdvisor(vectorStore, SearchRequest.defaults())) + .user(userText) + .call() + .chatResponse(); + + var relevancyEvaluator = new RelevancyEvaluator(ChatClient.builder(chatModel)); + + EvaluationRequest evaluationRequest = new EvaluationRequest(userText, + (List) response.getMetadata().get(QuestionAnswerAdvisor.RETRIEVED_DOCUMENTS), response); + + EvaluationResponse evaluationResponse = relevancyEvaluator.evaluate(evaluationRequest); + + assertTrue(evaluationResponse.isPass(), "Response is not relevant to the question"); + +} +---- + +The code above is from the example application located https://github.com/rd-1-2022/ai-azure-rag.git[here]. \ No newline at end of file diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfiguration.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfiguration.java index 17b7372b3..86d654945 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfiguration.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfiguration.java @@ -122,8 +122,8 @@ public class OpenAiAutoConfiguration { String baseUrl = StringUtils.hasText(imageProperties.getBaseUrl()) ? imageProperties.getBaseUrl() : commonProperties.getBaseUrl(); - Assert.hasText(apiKey, "OpenAI API key must be set"); - Assert.hasText(baseUrl, "OpenAI base URL must be set"); + Assert.hasText(apiKey, "OpenAI API key must be set. Use the property: spring.ai.openai.base-url"); + Assert.hasText(baseUrl, "OpenAI base URL must be set. Use the property: spring.ai.openai.api-key"); var openAiImageApi = new OpenAiImageApi(baseUrl, apiKey, restClientBuilder, responseErrorHandler);