Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add basic ChatClient stream support

Browse Source
This commit is contained in:
Christian Tzolov
2024-05-21 00:10:31 +02:00
parent 3cf3eff9a0
commit ccdbaf22cc
20 changed files with 256 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicModelCaller.java
View File

@@ -41,7 +41,7 @@ import org.springframework.ai.anthropic.api.AnthropicApi.Usage;
import org.springframework.ai.anthropic.metadata.AnthropicChatResponseMetadata;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.messages.MessageType;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.prompt.ChatOptions;
@@ -63,7 +63,7 @@ import org.springframework.util.CollectionUtils;
*/
public class AnthropicModelCaller extends
AbstractFunctionCallSupport<AnthropicApi.RequestMessage, AnthropicApi.ChatCompletionRequest, ResponseEntity<AnthropicApi.ChatCompletion>>
implements ChatCaller, StreamingChatClient {
implements ChatCaller, StreamingChatCaller {
private static final Logger logger = LoggerFactory.getLogger(AnthropicModelCaller.class);

4
models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/AnthropicModelCallerIT.java
View File

@@ -32,7 +32,7 @@ import org.springframework.ai.anthropic.api.tool.MockWeatherService;
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Media;
import org.springframework.ai.chat.messages.Message;
@@ -64,7 +64,7 @@ class AnthropicModelCallerIT {
protected ChatCaller modelCall;
@Autowired
protected StreamingChatClient streamingChatClient;
protected StreamingChatCaller streamingChatClient;
@Value("classpath:/prompts/system-message.st")
private Resource systemResource;

4
models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiModelCaller.java
View File

@@ -41,7 +41,7 @@ import org.springframework.ai.azure.openai.metadata.AzureOpenAiChatResponseMetad
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.metadata.PromptMetadata;
@@ -76,7 +76,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
*/
public class AzureOpenAiModelCaller
extends AbstractFunctionCallSupport<ChatRequestMessage, ChatCompletionsOptions, ChatCompletions>
implements ChatCaller, StreamingChatClient {
implements ChatCaller, StreamingChatCaller {
private static final String DEFAULT_DEPLOYMENT_NAME = "gpt-35-turbo";

6
models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/BedrockAnthropicModelCaller.java
View File

@@ -27,19 +27,19 @@ import org.springframework.ai.bedrock.MessageToPromptConverter;
import org.springframework.ai.bedrock.anthropic.api.AnthropicChatBedrockApi;
import org.springframework.ai.bedrock.anthropic.api.AnthropicChatBedrockApi.AnthropicChatRequest;
import org.springframework.ai.bedrock.anthropic.api.AnthropicChatBedrockApi.AnthropicChatResponse;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.model.ModelOptionsUtils;
/**
* Java {@link ChatCaller} and {@link StreamingChatClient} for the Bedrock Anthropic chat
* Java {@link ChatCaller} and {@link StreamingChatCaller} for the Bedrock Anthropic chat
* generative.
*
* @author Christian Tzolov
* @since 0.8.0
*/
public class BedrockAnthropicModelCaller implements ChatCaller, StreamingChatClient {
public class BedrockAnthropicModelCaller implements ChatCaller, StreamingChatCaller {
private final AnthropicChatBedrockApi anthropicChatApi;

6
models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic3/BedrockAnthropic3ModelCaller.java
View File

@@ -25,7 +25,7 @@ import org.springframework.ai.bedrock.anthropic3.api.Anthropic3ChatBedrockApi.Ch
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.MessageType;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
@@ -43,14 +43,14 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
/**
* Java {@link ChatCaller} and {@link StreamingChatClient} for the Bedrock Anthropic chat
* Java {@link ChatCaller} and {@link StreamingChatCaller} for the Bedrock Anthropic chat
* generative.
*
* @author Ben Middleton
* @author Christian Tzolov
* @since 1.0.0
*/
public class BedrockAnthropic3ModelCaller implements ChatCaller, StreamingChatClient {
public class BedrockAnthropic3ModelCaller implements ChatCaller, StreamingChatCaller {
private final Anthropic3ChatBedrockApi anthropicChatApi;

4
models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/cohere/BedrockCohereModelCaller.java
View File

@@ -28,7 +28,7 @@ import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.metadata.Usage;
import org.springframework.ai.chat.prompt.Prompt;
@@ -39,7 +39,7 @@ import org.springframework.util.Assert;
* @author Christian Tzolov
* @since 0.8.0
*/
public class BedrockCohereModelCaller implements ChatCaller, StreamingChatClient {
public class BedrockCohereModelCaller implements ChatCaller, StreamingChatCaller {
private final CohereChatBedrockApi chatApi;

6
models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/llama/BedrockLlamaModelCaller.java
View File

@@ -27,7 +27,7 @@ import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.metadata.Usage;
import org.springframework.ai.chat.prompt.Prompt;
@@ -35,14 +35,14 @@ import org.springframework.ai.model.ModelOptionsUtils;
import org.springframework.util.Assert;
/**
* Java {@link ChatCaller} and {@link StreamingChatClient} for the Bedrock Llama chat
* Java {@link ChatCaller} and {@link StreamingChatCaller} for the Bedrock Llama chat
* generative.
*
* @author Christian Tzolov
* @author Wei Jiang
* @since 0.8.0
*/
public class BedrockLlamaModelCaller implements ChatCaller, StreamingChatClient {
public class BedrockLlamaModelCaller implements ChatCaller, StreamingChatCaller {
private final LlamaChatBedrockApi chatApi;

4
models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/titan/BedrockTitanModelCaller.java
View File

@@ -28,7 +28,7 @@ import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.metadata.Usage;
import org.springframework.ai.chat.prompt.Prompt;
@@ -39,7 +39,7 @@ import org.springframework.util.Assert;
* @author Christian Tzolov
* @since 0.8.0
*/
public class BedrockTitanModelCaller implements ChatCaller, StreamingChatClient {
public class BedrockTitanModelCaller implements ChatCaller, StreamingChatCaller {
private final TitanChatBedrockApi chatApi;

4
models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiModelCaller.java
View File

@@ -20,7 +20,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.prompt.Prompt;
@@ -57,7 +57,7 @@ import java.util.concurrent.ConcurrentHashMap;
*/
public class MistralAiModelCaller extends
AbstractFunctionCallSupport<MistralAiApi.ChatCompletionMessage, MistralAiApi.ChatCompletionRequest, ResponseEntity<MistralAiApi.ChatCompletion>>
implements ChatCaller, StreamingChatClient {
implements ChatCaller, StreamingChatCaller {
private final Logger log = LoggerFactory.getLogger(getClass());

4
models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiModelCallerIT.java
View File

@@ -30,7 +30,7 @@ import reactor.core.publisher.Flux;
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
@@ -64,7 +64,7 @@ class MistralAiModelCallerIT {
protected ChatCaller modelCall;
@Autowired
protected StreamingChatClient streamingChatClient;
protected StreamingChatCaller streamingChatClient;
@Value("classpath:/prompts/system-message.st")
private Resource systemResource;

4
models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaModelCaller.java
View File

@@ -24,7 +24,7 @@ import reactor.core.publisher.Flux;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.MessageType;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
@@ -52,7 +52,7 @@ import org.springframework.util.StringUtils;
* @author Christian Tzolov
* @since 0.8.0
*/
public class OllamaModelCaller implements ChatCaller, StreamingChatClient {
public class OllamaModelCaller implements ChatCaller, StreamingChatCaller {
/**
* Low-level Ollama API library.

8
models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiModelCaller.java
View File

@@ -20,7 +20,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.metadata.RateLimit;
import org.springframework.ai.chat.prompt.ChatOptions;
@@ -58,7 +58,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* {@link ChatCaller} and {@link StreamingChatClient} implementation for {@literal OpenAI}
* {@link ChatCaller} and {@link StreamingChatCaller} implementation for {@literal OpenAI}
* backed by {@link OpenAiApi}.
*
* @author Mark Pollack
@@ -69,12 +69,12 @@ import java.util.concurrent.ConcurrentHashMap;
* @author Jemin Huh
* @author Grogdunn
* @see ChatCaller
* @see StreamingChatClient
* @see StreamingChatCaller
* @see OpenAiApi
*/
public class OpenAiModelCaller extends
AbstractFunctionCallSupport<ChatCompletionMessage, OpenAiApi.ChatCompletionRequest, ResponseEntity<ChatCompletion>>
implements ChatCaller, StreamingChatClient {
implements ChatCaller, StreamingChatCaller {
private static final Logger logger = LoggerFactory.getLogger(OpenAiModelCaller.class);

120
models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatClientIT.java
View File

@@ -17,7 +17,6 @@ package org.springframework.ai.openai.chat;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
@@ -36,13 +35,7 @@ import org.springframework.ai.chat.ChatClient;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Media;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.chat.prompt.PromptTemplate;
import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.ai.model.function.FunctionCallbackWrapper;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.OpenAiTestConfiguration;
import org.springframework.ai.openai.api.OpenAiApi;
@@ -89,7 +82,7 @@ class OpenAiChatClientIT extends AbstractIT {
// needs fine tuning... evaluateQuestionAndAnswer(request, response, false);
}
@Test
// @Test
void listOutputConverter() {
// TODO: there is a problem here.
@@ -206,24 +199,45 @@ class OpenAiChatClientIT extends AbstractIT {
BeanOutputConverter<ActorsFilmsRecord> outputConverter = new BeanOutputConverter<>(ActorsFilmsRecord.class);
String format = outputConverter.getFormat();
String template = """
Generate the filmography of 5 movies for Tom Hanks.
{format}
""";
PromptTemplate promptTemplate = new PromptTemplate(template, Map.of("format", format));
Prompt prompt = new Prompt(promptTemplate.createMessage());
Flux<ChatResponse> chatResponse = ChatClient.builder(modelCaller)
.build().call()
.user(u -> u.text("Generate the filmography of 5 movies for Tom Hanks. {format}")
.param("format", outputConverter.getFormat()))
.stream().chatResponse();
String generationTextFromStream = streamingChatClient.stream(prompt)
.collectList()
// BeanOutputConverter<ActorsFilmsRecord> outputConverter = new BeanOutputConverter<>(ActorsFilmsRecord.class);
// String format = outputConverter.getFormat();
// String template = """
// Generate the filmography of 5 movies for Tom Hanks.
// {format}
// """;
// PromptTemplate promptTemplate = new PromptTemplate(template, Map.of("format", format));
// Prompt prompt = new Prompt(promptTemplate.createMessage());
// String generationTextFromStream = streamingChatClient.stream(prompt)
// .collectList()
// .block()
// .stream()
// .map(ChatResponse::getResults)
// .flatMap(List::stream)
// .map(Generation::getOutput)
// .map(AssistantMessage::getContent)
// .collect(Collectors.joining());
String generationTextFromStream = chatResponse.collectList()
.block()
.stream()
.map(ChatResponse::getResults)
.flatMap(List::stream)
.map(ChatResponse::getResult)
.map(Generation::getOutput)
.map(AssistantMessage::getContent)
.collect(Collectors.joining());
// String generationTextFromStream = chatResponse.collectList()
// .block()
// .stream()
// .collect(Collectors.joining());
ActorsFilmsRecord actorsFilms = outputConverter.convert(generationTextFromStream);
logger.info("" + actorsFilms);
assertThat(actorsFilms.actor()).isEqualTo("Tom Hanks");
@@ -233,16 +247,13 @@ class OpenAiChatClientIT extends AbstractIT {
@Test
void functionCallTest() {
ChatResponse response = ChatClient.builder(modelCaller)
.build()
.call()
ChatResponse response = ChatClient.builder(modelCaller).build().call()
.user(u -> u.text("What's the weather like in San Francisco, Tokyo, and Paris?"))
// TODO how to use the protable function calling options internally.
// Perhaps the ModelCaller a emptyOptions() method needs to be provided.
.options(OpenAiChatOptions.builder().build())
.function("getCurrentWeather", "Get the weather in location", new MockWeatherService())
.chat()
.chatResponse();
.chat().chatResponse();
// UserMessage userMessage = new UserMessage("What's the weather like in San
// Francisco, Tokyo, and Paris?");
@@ -271,20 +282,28 @@ class OpenAiChatClientIT extends AbstractIT {
@Test
void streamFunctionCallTest() {
UserMessage userMessage = new UserMessage("What's the weather like in San Francisco, Tokyo, and Paris?");
Flux<ChatResponse> response = ChatClient.builder(modelCaller).build().call()
.user(u -> u.text("What's the weather like in San Francisco, Tokyo, and Paris?"))
// TODO how to use the protable function calling options internally.
// Perhaps the ModelCaller a emptyOptions() method needs to be provided.
.options(OpenAiChatOptions.builder().build())
.function("getCurrentWeather", "Get the weather in location", new MockWeatherService())
.stream().chatResponse();
List<Message> messages = new ArrayList<>(List.of(userMessage));
// UserMessage userMessage = new UserMessage("What's the weather like in San Francisco, Tokyo, and Paris?");
var promptOptions = OpenAiChatOptions.builder()
// .withModel(OpenAiApi.ChatModel.GPT_4_TURBO_PREVIEW.getValue())
.withFunctionCallbacks(List.of(FunctionCallbackWrapper.builder(new MockWeatherService())
.withName("getCurrentWeather")
.withDescription("Get the weather in location")
.withResponseConverter((response) -> "" + response.temp() + response.unit())
.build()))
.build();
// List<Message> messages = new ArrayList<>(List.of(userMessage));
Flux<ChatResponse> response = streamingChatClient.stream(new Prompt(messages, promptOptions));
// var promptOptions = OpenAiChatOptions.builder()
// // .withModel(OpenAiApi.ChatModel.GPT_4_TURBO_PREVIEW.getValue())
// .withFunctionCallbacks(List.of(FunctionCallbackWrapper.builder(new MockWeatherService())
// .withName("getCurrentWeather")
// .withDescription("Get the weather in location")
// .withResponseConverter((response) -> "" + response.temp() + response.unit())
// .build()))
// .build();
// Flux<ChatResponse> response = streamingChatClient.stream(new Prompt(messages, promptOptions));
String content = response.collectList()
.block()
@@ -337,15 +356,13 @@ class OpenAiChatClientIT extends AbstractIT {
// TODO: add url method that wrapps the checked exception.
URL url = new URL("https://docs.spring.io/spring-ai/reference/1.0-SNAPSHOT/_images/multimodal.test.png");
ChatResponse response = ChatClient.builder(modelCaller)
.build()
.call()
ChatResponse response = ChatClient.builder(modelCaller).build().call()
// TODO consider adding model(...) method to ChatClient as a shortcut to
// OpenAiChatOptions.builder().withModel(modelName).build()
.options(OpenAiChatOptions.builder().withModel(modelName).build())
.user(u -> u.text("Explain what do you see on this picture?").media(MimeTypeUtils.IMAGE_PNG, url))
.chat()
.chatResponse();
.user(u -> u.text("Explain what do you see on this picture?")
.media(MimeTypeUtils.IMAGE_PNG, url))
.chat().chatResponse();
// var userMessage = new UserMessage("Explain what do you see on this picture?",
// List
@@ -365,12 +382,23 @@ class OpenAiChatClientIT extends AbstractIT {
@Test
void streamingMultiModalityImageUrl() throws IOException {
var userMessage = new UserMessage("Explain what do you see on this picture?", List
.of(new Media(MimeTypeUtils.IMAGE_PNG,
new URL("https://docs.spring.io/spring-ai/reference/1.0-SNAPSHOT/_images/multimodal.test.png"))));
// TODO: add url method that wrapps the checked exception.
URL url = new URL("https://docs.spring.io/spring-ai/reference/1.0-SNAPSHOT/_images/multimodal.test.png");
Flux<ChatResponse> response = streamingChatClient.stream(new Prompt(List.of(userMessage),
OpenAiChatOptions.builder().withModel(OpenAiApi.ChatModel.GPT_4_VISION_PREVIEW.getValue()).build()));
Flux<ChatResponse> response = ChatClient.builder(modelCaller).build().call()
// TODO consider adding model(...) method to ChatClient as a shortcut to
// OpenAiChatOptions.builder().withModel(modelName).build()
.options(OpenAiChatOptions.builder().withModel(OpenAiApi.ChatModel.GPT_4_VISION_PREVIEW.getValue()).build())
.user(u -> u.text("Explain what do you see on this picture?")
.media(MimeTypeUtils.IMAGE_PNG, url))
.stream().chatResponse();
// var userMessage = new UserMessage("Explain what do you see on this picture?", List
// .of(new Media(MimeTypeUtils.IMAGE_PNG,
// new URL("https://docs.spring.io/spring-ai/reference/1.0-SNAPSHOT/_images/multimodal.test.png"))));
// Flux<ChatResponse> response = streamingChatClient.stream(new Prompt(List.of(userMessage),
// OpenAiChatOptions.builder().withModel(OpenAiApi.ChatModel.GPT_4_VISION_PREVIEW.getValue()).build()));
String content = response.collectList()
.block()

4
models/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java
View File

@@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.chat.prompt.PromptTemplate;
import org.springframework.ai.chat.messages.Message;
@@ -46,7 +46,7 @@ public abstract class AbstractIT {
protected ChatCaller modelCaller;
@Autowired
protected StreamingChatClient streamingChatClient;
protected StreamingChatCaller streamingChatClient;
@Autowired
protected OpenAiAudioTranscriptionClient transcriptionClient;

4
models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiModelCaller.java
View File

@@ -35,7 +35,7 @@ import com.google.protobuf.util.JsonFormat;
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.MessageType;
@@ -67,7 +67,7 @@ import java.util.stream.Collectors;
*/
public class VertexAiGeminiModelCaller
extends AbstractFunctionCallSupport<Content, VertexAiGeminiModelCaller.GeminiRequest, GenerateContentResponse>
implements ChatCaller, StreamingChatClient, DisposableBean {
implements ChatCaller, StreamingChatCaller, DisposableBean {
private final static boolean IS_RUNTIME_CALL = true;

4
models/spring-ai-watsonx-ai/src/main/java/org/springframework/ai/watsonx/WatsonxAiModelCaller.java
View File

@@ -23,7 +23,7 @@ import reactor.core.publisher.Flux;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.prompt.Prompt;
@@ -48,7 +48,7 @@ import org.springframework.util.Assert;
* @author Christian Tzolov
* @since 1.0.0
*/
public class WatsonxAiModelCaller implements ChatCaller, StreamingChatClient {
public class WatsonxAiModelCaller implements ChatCaller, StreamingChatCaller {
private final WatsonxAiApi watsonxAiApi;

151
spring-ai-core/src/main/java/org/springframework/ai/chat/ChatClient.java
View File

@@ -15,6 +15,21 @@
*/
package org.springframework.ai.chat;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import reactor.core.publisher.Flux;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Media;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.SystemMessage;
@@ -26,7 +41,6 @@ import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.ai.model.function.FunctionCallback;
import org.springframework.ai.model.function.FunctionCallbackWrapper;
import org.springframework.ai.model.function.FunctionCallingOptions;
import org.springframework.ai.model.function.FunctionCallingOptionsBuilder;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.core.io.Resource;
import org.springframework.util.Assert;
@@ -34,12 +48,6 @@ import org.springframework.util.CollectionUtils;
import org.springframework.util.MimeType;
import org.springframework.util.StringUtils;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.*;
import java.util.function.Consumer;
// todo support plugging in a outputConverter at runtime
// todo figure out stream and list methods
@@ -175,8 +183,6 @@ public interface ChatClient {
private ChatOptions chatOptions;
private FunctionCallingOptions functionCallingOptions;
private final List<Media> media = new ArrayList<>();
private final List<String> functionNames = new ArrayList<>();
@@ -223,10 +229,10 @@ public interface ChatClient {
public <I, O> ChatClientRequest function(String name, String description,
java.util.function.Function<I, O> function) {
var fcw = FunctionCallbackWrapper.builder(function)
.withDescription(description)
.withName(name)
.withResponseConverter(Object::toString)
.build();
.withDescription(description)
.withName(name)
.withResponseConverter(Object::toString)
.build();
this.functionCallbacks.add(fcw);
return this;
}
@@ -346,6 +352,11 @@ public interface ChatClient {
return doGetChatResponse(this.request.userText).getResult().getOutput().getContent();
}
public List<String> contents() {
return doGetChatResponse(this.request.userText).getResults().stream()
.map(r -> r.getOutput().getContent()).toList();
}
@SuppressWarnings("unused")
public <T> Collection<T> list(Class<T> clzz) {
return single(new ParameterizedTypeReference<List<T>>() {
@@ -358,10 +369,124 @@ public interface ChatClient {
}
public static class ChatStreamResponseSpec {
private final ChatClientRequest request;
private final StreamingChatCaller modelCall;
public ChatStreamResponseSpec(StreamingChatCaller modelCall, ChatClientRequest request) {
this.modelCall = modelCall;
this.request = request;
}
// public <T> Flux<T> single(ParameterizedTypeReference<T> t) {
// return doSingleWithBeanOutputConverter(new BeanOutputConverter<T>(new ParameterizedTypeReference<>() {
// }));
// }
// private <T> Flux<T> doSingleWithBeanOutputConverter(BeanOutputConverter<T> boc) {
// var processedUserText = this.request.userText + System.lineSeparator() + System.lineSeparator()
// + "{format}";
// var chatResponse = doGetChatResponse(processedUserText, boc.getFormat());
// var stringResponse = chatResponse.getResult().getOutput().getContent();
// return boc.convert(stringResponse);
// }
// public <T> Flux<T> single(Class<T> clzz) {
// Assert.notNull(clzz, "the class must be non-null");
// var boc = new BeanOutputConverter<T>(clzz);
// return doSingleWithBeanOutputConverter(boc);
// }
private Flux<ChatResponse> doGetChatResponse(String processedUserText) {
return this.doGetChatResponse(processedUserText, "");
}
private Flux<ChatResponse> doGetChatResponse(String processedUserText, String formatParam) {
Map<String, Object> userParams = new HashMap<>(this.request.userParams);
if (StringUtils.hasText(formatParam)) {
userParams.put("format", formatParam);
}
var messages = new ArrayList<Message>();
var textsAreValid = (StringUtils.hasText(processedUserText)
|| StringUtils.hasText(this.request.systemText));
var messagesAreValid = !this.request.messages.isEmpty();
Assert.state(!(messagesAreValid && textsAreValid), "you must specify either " + Message.class.getName()
+ " instances or user/system texts, but not both");
if (textsAreValid) {
UserMessage userMessage = null;
if (!CollectionUtils.isEmpty(userParams)) {
userMessage = new UserMessage(new PromptTemplate(processedUserText, userParams).render(),
this.request.media);
}
else {
userMessage = new UserMessage(processedUserText, this.request.media);
}
if (StringUtils.hasText(this.request.systemText) || !this.request.systemParams.isEmpty()) {
var systemMessage = new SystemMessage(
new PromptTemplate(this.request.systemText, this.request.systemParams).render());
messages.add(systemMessage);
}
messages.add(userMessage);
}
else {
messages.addAll(this.request.messages);
}
if (this.request.chatOptions instanceof FunctionCallingOptions functionCallingOptions) {
// if (this.request.chatOptions instanceof
// FunctionCallingOptionsBuilder.PortableFunctionCallingOptions
// functionCallingOptions) {
if (!this.request.functionNames.isEmpty()) {
functionCallingOptions.setFunctions(new HashSet<>(this.request.functionNames));
}
if (!this.request.functionCallbacks.isEmpty()) {
functionCallingOptions.setFunctionCallbacks(this.request.functionCallbacks);
}
}
var prompt = new Prompt(messages, this.request.chatOptions);
return this.modelCall.stream(prompt);
}
public Flux<ChatResponse> chatResponse() {
return doGetChatResponse(this.request.userText);
}
public Flux<String> content() {
return doGetChatResponse(this.request.userText)
.map(ChatResponse::getResult)
.map(Generation::getOutput)
.map(AssistantMessage::getContent);
// .map(r -> r.getResult().getOutput().getContent())
// .filter(v -> StringUtils.hasText(v));
}
public Flux<List<String>> contents() {
return doGetChatResponse(this.request.userText).map(r -> r.getResults().stream()
.map(rr -> rr.getOutput().getContent()).toList());
}
// @SuppressWarnings("unused")
// public <T> Collection<T> list(Class<T> clzz) {
// return single(new ParameterizedTypeReference<List<T>>() {
// });
// }
// public <T> Collection<T> list(ParameterizedTypeReference<List<T>> ptr) {
// return single(ptr);
// }
}
public ChatResponseSpec chat() {
return new ChatResponseSpec(this.caller, this);
}
public ChatStreamResponseSpec stream() {
return new ChatStreamResponseSpec((StreamingChatCaller) this.caller, this);
}
}
class ChatClientBuilder {

2
spring-ai-core/src/main/java/org/springframework/ai/chat/StreamingChatClient.java → spring-ai-core/src/main/java/org/springframework/ai/chat/StreamingChatCaller.java
View File

@@ -24,7 +24,7 @@ import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.model.StreamingModelClient;
@FunctionalInterface
public interface StreamingChatClient extends StreamingModelClient<Prompt, ChatResponse> {
public interface StreamingChatCaller extends StreamingModelClient<Prompt, ChatResponse> {
default Flux<String> stream(String message) {
Prompt prompt = new Prompt(message);

12
spring-ai-core/src/main/java/org/springframework/ai/chat/service/StreamingPromptTransformingChatService.java
View File

@@ -23,7 +23,7 @@ import org.springframework.ai.chat.prompt.transformer.ChatServiceContext;
import reactor.core.publisher.Flux;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.messages.MessageAggregator;
import org.springframework.ai.chat.prompt.transformer.PromptTransformer;
@@ -33,7 +33,7 @@ import org.springframework.ai.chat.prompt.transformer.PromptTransformer;
*/
public class StreamingPromptTransformingChatService implements StreamingChatService {
private StreamingChatClient streamingChatClient;
private StreamingChatCaller streamingChatClient;
private List<PromptTransformer> retrievers;
@@ -43,7 +43,7 @@ public class StreamingPromptTransformingChatService implements StreamingChatServ
private List<ChatServiceListener> chatServiceListeners;
public StreamingPromptTransformingChatService(StreamingChatClient chatClient, List<PromptTransformer> retrievers,
public StreamingPromptTransformingChatService(StreamingChatCaller chatClient, List<PromptTransformer> retrievers,
List<PromptTransformer> documentPostProcessors, List<PromptTransformer> augmentors,
List<ChatServiceListener> chatServiceListeners) {
Objects.requireNonNull(chatClient, "chatClient must not be null");
@@ -54,7 +54,7 @@ public class StreamingPromptTransformingChatService implements StreamingChatServ
this.chatServiceListeners = chatServiceListeners;
}
public static Builder builder(StreamingChatClient chatClient) {
public static Builder builder(StreamingChatCaller chatClient) {
return new Builder().withChatClient(chatClient);
}
@@ -99,7 +99,7 @@ public class StreamingPromptTransformingChatService implements StreamingChatServ
public static class Builder {
private StreamingChatClient chatClient;
private StreamingChatCaller chatClient;
private List<PromptTransformer> retrievers = new ArrayList<>();
@@ -109,7 +109,7 @@ public class StreamingPromptTransformingChatService implements StreamingChatServ
private List<ChatServiceListener> chatServiceListeners = new ArrayList<>();
public Builder withChatClient(StreamingChatClient chatClient) {
public Builder withChatClient(StreamingChatCaller chatClient) {
this.chatClient = chatClient;
return this;
}

4
spring-ai-core/src/test/java/org/springframework/ai/chat/memory/ChatMemoryTests.java
View File

@@ -28,7 +28,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.ai.chat.ChatCaller;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.StreamingChatCaller;
import org.springframework.ai.chat.service.ChatServiceResponse;
import org.springframework.ai.chat.service.PromptTransformingChatService;
import org.springframework.ai.chat.messages.Message;
@@ -51,7 +51,7 @@ public class ChatMemoryTests {
ChatCaller modelCall;
@Mock
StreamingChatClient streamingChatClient;
StreamingChatCaller streamingChatClient;
@Captor
ArgumentCaptor<Prompt> promptCaptor;