diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chatclient.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chatclient.adoc index 1cb59b3eb..67f685055 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chatclient.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chatclient.adoc @@ -47,23 +47,170 @@ class MyController { In this simple example, the user input sets the contents of the user message. The `call()` method sends a request to the AI model, and the `content()` method returns the AI model's response as a `String`. -=== Create a ChatClient programmatically +=== Working with Multiple Chat Models -You can disable the `ChatClient.Builder` autoconfiguration by setting the property `spring.ai.chat.client.enabled=false`. -This is useful if multiple chat models are used together. -Then, create a `ChatClient.Builder` instance programmatically for every `ChatModel` you need: +There are several scenarios where you might need to work with multiple chat models in a single application: + +* Using different models for different types of tasks (e.g., a powerful model for complex reasoning and a faster, cheaper model for simpler tasks) +* Implementing fallback mechanisms when one model service is unavailable +* A/B testing different models or configurations +* Providing users with a choice of models based on their preferences +* Combining specialized models (one for code generation, another for creative content, etc.) + +By default, Spring AI autoconfigures a single `ChatClient.Builder` bean. However, you may need to work with multiple chat models in your application. Here's how to handle this scenario: + +In all cases, you need to disable the `ChatClient.Builder` autoconfiguration by setting the property `spring.ai.chat.client.enabled=false`. + +This allows you to create multiple `ChatClient` instances manually. + +==== Multiple ChatClients with a Single Model Type + +This section covers a common use case where you need to create multiple ChatClient instances that all use the same underlying model type but with different configurations. [source,java] ---- -ChatModel myChatModel = ... // usually autowired +// Create ChatClient instances programmatically +ChatModel myChatModel = ... // already autoconfigured by Spring Boot +ChatClient chatClient = ChatClient.create(myChatModel); -ChatClient.Builder builder = ChatClient.builder(this.myChatModel); - -// or create a ChatClient with the default builder settings: - -ChatClient chatClient = ChatClient.create(this.myChatModel); +// Or use the builder for more control +ChatClient.Builder builder = ChatClient.builder(myChatModel); +ChatClient customChatClient = builder + .defaultSystemPrompt("You are a helpful assistant.") + .build(); ---- +==== ChatClients for Different Model Types + +When working with multiple AI models, you can define separate `ChatClient` beans for each model: + +[source,java] +---- +import org.springframework.ai.chat.ChatClient; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ChatClientConfig { + + @Bean + public ChatClient openAiChatClient(OpenAiChatModel chatModel) { + return ChatClient.create(chatModel); + } + + @Bean + public ChatClient anthropicChatClient(AnthropicChatModel chatModel) { + return ChatClient.create(chatModel); + } +} +---- + +You can then inject these beans into your application components using the `@Qualifier` annotation: + +[source,java] +---- + +@Configuration +public class ChatClientExample { + + @Bean + CommandLineRunner cli( + @Qualifier("openAiChatClient") ChatClient openAiChatClient, + @Qualifier("anthropicChatClient") ChatClient anthropicChatClient) { + + return args -> { + var scanner = new Scanner(System.in); + ChatClient chat; + + // Model selection + System.out.println("\nSelect your AI model:"); + System.out.println("1. OpenAI"); + System.out.println("2. Anthropic"); + System.out.print("Enter your choice (1 or 2): "); + + String choice = scanner.nextLine().trim(); + + if (choice.equals("1")) { + chat = openAiChatClient; + System.out.println("Using OpenAI model"); + } else { + chat = anthropicChatClient; + System.out.println("Using Anthropic model"); + } + + // Use the selected chat client + System.out.print("\nEnter your question: "); + String input = scanner.nextLine(); + String response = chat.prompt(input).call().content(); + System.out.println("ASSISTANT: " + response); + + scanner.close(); + }; + } +} +---- + +==== Multiple OpenAI-Compatible API Endpoints + +The `OpenAiApi` and `OpenAiChatModel` classes provide a `mutate()` method that allows you to create variations of existing instances with different properties. This is particularly useful when you need to work with multiple OpenAI-compatible APIs. + +[source,java] +---- + +@Service +public class MultiModelService { + + private static final Logger logger = LoggerFactory.getLogger(MultiModelService.class); + + @Autowired + private OpenAiChatModel baseChatModel; + + @Autowired + private OpenAiApi baseOpenAiApi; + + public void multiClientFlow() { + try { + // Derive a new OpenAiApi for Groq (Llama3) + OpenAiApi groqApi = baseOpenAiApi.mutate() + .baseUrl("https://api.groq.com/openai") + .apiKey(System.getenv("GROQ_API_KEY")) + .build(); + + // Derive a new OpenAiApi for OpenAI GPT-4 + OpenAiApi gpt4Api = baseOpenAiApi.mutate() + .baseUrl("https://api.openai.com") + .apiKey(System.getenv("OPENAI_API_KEY")) + .build(); + + // Derive a new OpenAiChatModel for Groq + OpenAiChatModel groqModel = baseChatModel.mutate() + .openAiApi(groqApi) + .defaultOptions(OpenAiChatOptions.builder().model("llama3-70b-8192").temperature(0.5).build()) + .build(); + + // Derive a new OpenAiChatModel for GPT-4 + OpenAiChatModel gpt4Model = baseChatModel.mutate() + .openAiApi(gpt4Api) + .defaultOptions(OpenAiChatOptions.builder().model("gpt-4").temperature(0.7).build()) + .build(); + + // Simple prompt for both models + String prompt = "What is the capital of France?"; + + String groqResponse = ChatClient.builder(groqModel).build().prompt(prompt).call().content(); + String gpt4Response = ChatClient.builder(gpt4Model).build().prompt(prompt).call().content(); + + logger.info("Groq (Llama3) response: {}", groqResponse); + logger.info("OpenAI GPT-4 response: {}", gpt4Response); + } + catch (Exception e) { + logger.error("Error in multi-client flow", e); + } + } +} +---- + + == ChatClient Fluent API The `ChatClient` fluent API allows you to create a prompt in three distinct ways using an overloaded `prompt` method to initiate the fluent API: @@ -407,72 +554,6 @@ In this configuration, the `MessageChatMemoryAdvisor` will be executed first, ad xref:ROOT:api/retrieval-augmented-generation.adoc#_questionansweradvisor[Learn about Question Answer Advisor] -TIP: Refer to the xref:api/chat-memory.adoc[Chat Memory] documentation for more information on how to use the `ChatMemory` interface to manage conversation history in combination with the advisors. - -The following advisor implementations use the `ChatMemory` interface to advice the prompt with conversation history which differ in the details of how the memory is added to the prompt - -* `MessageChatMemoryAdvisor` : Memory is retrieved and added as a collection of messages to the prompt -* `PromptChatMemoryAdvisor` : Memory is retrieved and added into the prompt's system text. -* `VectorStoreChatMemoryAdvisor` : The constructor `VectorStoreChatMemoryAdvisor(VectorStore vectorStore, String defaultConversationId, int chatHistoryWindowSize, int order)` This constructor allows you to: - -. Specify the VectorStore instance used for managing and querying documents. -. Set a default conversation ID to be used if none is provided in the context. -. Define the window size for chat history retrieval in terms of token size. -. Provide system text advice used for the chat advisor system. -. Set the order of precedence for this advisor in the chain. - - -The `VectorStoreChatMemoryAdvisor.builder()` method lets you specify the default conversation ID, the chat history window size, and the order of the chat history to be retrieved. - -A sample `@Service` implementation that uses several advisors is shown below. - -[source,java] ----- -import static org.springframework.ai.chat.memory.ChatMemory.CHAT_MEMORY_CONVERSATION_ID_KEY; -import static org.springframework.ai.chat.client.advisor.vectorstore.VectorStoreChatMemoryAdvisor.CHAT_MEMORY_RETRIEVE_SIZE_KEY; - -@Service -public class CustomerSupportAssistant { - - private final ChatClient chatClient; - - public CustomerSupportAssistant(ChatClient.Builder builder, VectorStore vectorStore, ChatMemory chatMemory) { - - this.chatClient = builder - .defaultSystem(""" - You are a customer chat support agent of an airline named "Funnair". Respond in a friendly, - helpful, and joyful manner. - - Before providing information about a booking or cancelling a booking, you MUST always - get the following information from the user: booking number, customer first name and last name. - - Before changing a booking you MUST ensure it is permitted by the terms. - - If there is a charge for the change, you MUST ask the user to consent before proceeding. - """) - .defaultAdvisors( - new MessageChatMemoryAdvisor(chatMemory), // CHAT MEMORY - new QuestionAnswerAdvisor(vectorStore), // RAG - new SimpleLoggerAdvisor()) - .defaultFunctions("getBookingDetails", "changeBooking", "cancelBooking") // FUNCTION CALLING - .build(); - } - - public Flux chat(String chatId, String userMessageContent) { - - return this.chatClient.prompt() - .user(userMessageContent) - .advisors(a -> a - .param(CHAT_MEMORY_CONVERSATION_ID_KEY, chatId) - .param(CHAT_MEMORY_RETRIEVE_SIZE_KEY, 100)) - .stream().content(); - } - -} ----- - -xref:ROOT:api/retrieval-augmented-generation.adoc#_questionansweradvisor[Learn about Question Answer Advisor] - === Retrieval Augmented Generation Refer to the xref:ROOT:api/retrieval-augmented-generation.adoc[Retrieval Augmented Generation] guide.