Add Bedrock Anthropic Chat Options support
- Improve Anthropic tests - Add anthrpic docs - Restructure the docs for Azure OpenAI, OpenAI, Ollama, Bedrock Cohere and Bedrock Lllam2
This commit is contained in:
Christian Tzolov
3
models/spring-ai-azure-openai/README.md
Normal file
3
models/spring-ai-azure-openai/README.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# Azure OpenAI
|
||||
|
||||
Visit the Spring AI [Azure OpenAI Chat Documentation](https://docs.spring.io/spring-ai/reference/api/clients/azure-openai-chat.html).
|
||||
@@ -1,9 +0,0 @@
|
||||
# Azure OpenAI
|
||||
|
||||
Provides Azure OpenAI Chat and Embedding clients.
|
||||
Leverages the native [OpenAIClient](https://learn.microsoft.com/en-us/java/api/overview/azure/ai-openai-readme?view=azure-java-preview#streaming-chat-completions) to interact with the [Amazon AI Studio models and deployment](https://oai.azure.com/).
|
||||
|
||||
Find additional information:
|
||||
|
||||
- [Azure OpenAi Chat Client](https://docs.spring.io/spring-ai/reference/api/clients/azure-openai-chat.html)
|
||||
- [Azure OpenAi Embeddings Client](https://docs.spring.io/spring-ai/reference/api/embeddings/azure-openai-embeddings.html)
|
||||
@@ -78,14 +78,22 @@ public class AzureOpenAiChatClient implements ChatClient, StreamingChatClient {
|
||||
private final OpenAIClient openAIClient;
|
||||
|
||||
public AzureOpenAiChatClient(OpenAIClient microsoftOpenAiClient) {
|
||||
Assert.notNull(microsoftOpenAiClient, "com.azure.ai.openai.OpenAIClient must not be null");
|
||||
this.openAIClient = microsoftOpenAiClient;
|
||||
this.defaultOptions = AzureOpenAiChatOptions.builder()
|
||||
.withModel(DEFAULT_MODEL)
|
||||
.withTemperature(DEFAULT_TEMPERATURE)
|
||||
.build();
|
||||
this(microsoftOpenAiClient,
|
||||
AzureOpenAiChatOptions.builder().withModel(DEFAULT_MODEL).withTemperature(DEFAULT_TEMPERATURE).build());
|
||||
}
|
||||
|
||||
public AzureOpenAiChatClient(OpenAIClient microsoftOpenAiClient, AzureOpenAiChatOptions options) {
|
||||
Assert.notNull(microsoftOpenAiClient, "com.azure.ai.openai.OpenAIClient must not be null");
|
||||
Assert.notNull(options, "AzureOpenAiChatOptions must not be null");
|
||||
this.openAIClient = microsoftOpenAiClient;
|
||||
this.defaultOptions = options;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated since 0.8.0, use
|
||||
* {@link #AzureOpenAiChatClient(OpenAIClient, AzureOpenAiChatOptions)} instead.
|
||||
*/
|
||||
@Deprecated(forRemoval = true, since = "0.8.0")
|
||||
public AzureOpenAiChatClient withDefaultOptions(AzureOpenAiChatOptions defaultOptions) {
|
||||
Assert.notNull(defaultOptions, "DefaultOptions must not be null");
|
||||
this.defaultOptions = defaultOptions;
|
||||
|
||||
@@ -33,7 +33,7 @@ public class AzureChatCompletionsOptionsTests {
|
||||
public void createRequestWithChatOptions() {
|
||||
|
||||
OpenAIClient mockClient = Mockito.mock(OpenAIClient.class);
|
||||
var client = new AzureOpenAiChatClient(mockClient).withDefaultOptions(
|
||||
var client = new AzureOpenAiChatClient(mockClient,
|
||||
AzureOpenAiChatOptions.builder().withModel("DEFAULT_MODEL").withTemperature(66.6f).build());
|
||||
|
||||
var requestOptions = client.toAzureChatCompletionsOptions(new Prompt("Test message content"));
|
||||
|
||||
@@ -180,7 +180,7 @@ class AzureOpenAiChatClientIT {
|
||||
|
||||
@Bean
|
||||
public AzureOpenAiChatClient azureOpenAiChatClient(OpenAIClient openAIClient) {
|
||||
return new AzureOpenAiChatClient(openAIClient).withDefaultOptions(
|
||||
return new AzureOpenAiChatClient(openAIClient,
|
||||
AzureOpenAiChatOptions.builder().withModel("gpt-35-turbo").withMaxTokens(200).build());
|
||||
|
||||
}
|
||||
|
||||
@@ -194,7 +194,11 @@ public class AnthropicChatBedrockApi extends
|
||||
/**
|
||||
* anthropic.claude-v2
|
||||
*/
|
||||
CLAUDE_V2("anthropic.claude-v2");
|
||||
CLAUDE_V2("anthropic.claude-v2"),
|
||||
/**
|
||||
* anthropic.claude-v2:1
|
||||
*/
|
||||
CLAUDE_V21("anthropic.claude-v2:1");
|
||||
|
||||
private final String id;
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
* Copyright 2023-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -18,8 +18,6 @@ package org.springframework.ai.bedrock.cohere;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.ai.chat.ChatResponse;
|
||||
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import org.springframework.ai.bedrock.BedrockUsage;
|
||||
@@ -27,14 +25,16 @@ import org.springframework.ai.bedrock.MessageToPromptConverter;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatResponse;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.LogitBias;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.ReturnLikelihoods;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.Truncate;
|
||||
import org.springframework.ai.chat.ChatClient;
|
||||
import org.springframework.ai.chat.StreamingChatClient;
|
||||
import org.springframework.ai.chat.ChatOptions;
|
||||
import org.springframework.ai.chat.ChatResponse;
|
||||
import org.springframework.ai.chat.Generation;
|
||||
import org.springframework.ai.chat.StreamingChatClient;
|
||||
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
|
||||
import org.springframework.ai.chat.metadata.Usage;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.model.ModelOptionsUtils;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
@@ -44,71 +44,18 @@ public class BedrockCohereChatClient implements ChatClient, StreamingChatClient
|
||||
|
||||
private final CohereChatBedrockApi chatApi;
|
||||
|
||||
private Float temperature;
|
||||
|
||||
private Float topP;
|
||||
|
||||
private Integer topK;
|
||||
|
||||
private Integer maxTokens;
|
||||
|
||||
private List<String> stopSequences;
|
||||
|
||||
private ReturnLikelihoods returnLikelihoods;
|
||||
|
||||
private Integer numGenerations;
|
||||
|
||||
private LogitBias logitBias;
|
||||
|
||||
private Truncate truncate;
|
||||
private final BedrockCohereChatOptions defaultOptions;
|
||||
|
||||
public BedrockCohereChatClient(CohereChatBedrockApi chatApi) {
|
||||
this(chatApi, BedrockCohereChatOptions.builder().build());
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient(CohereChatBedrockApi chatApi, BedrockCohereChatOptions options) {
|
||||
Assert.notNull(chatApi, "CohereChatBedrockApi must not be null");
|
||||
Assert.notNull(options, "BedrockCohereChatOptions must not be null");
|
||||
|
||||
this.chatApi = chatApi;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withTemperature(Float temperature) {
|
||||
this.temperature = temperature;
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withTopP(Float topP) {
|
||||
this.topP = topP;
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withTopK(Integer topK) {
|
||||
this.topK = topK;
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withMaxTokens(Integer maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withStopSequences(List<String> stopSequences) {
|
||||
this.stopSequences = stopSequences;
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withReturnLikelihoods(ReturnLikelihoods returnLikelihoods) {
|
||||
this.returnLikelihoods = returnLikelihoods;
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withNumGenerations(Integer numGenerations) {
|
||||
this.numGenerations = numGenerations;
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withLogitBias(LogitBias logitBias) {
|
||||
this.logitBias = logitBias;
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatClient withTruncate(Truncate truncate) {
|
||||
this.truncate = truncate;
|
||||
return this;
|
||||
this.defaultOptions = options;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -134,21 +81,38 @@ public class BedrockCohereChatClient implements ChatClient, StreamingChatClient
|
||||
});
|
||||
}
|
||||
|
||||
private CohereChatRequest createRequest(Prompt prompt, boolean stream) {
|
||||
/**
|
||||
* Test access.
|
||||
*/
|
||||
CohereChatRequest createRequest(Prompt prompt, boolean stream) {
|
||||
final String promptValue = MessageToPromptConverter.create().toPrompt(prompt.getInstructions());
|
||||
|
||||
return CohereChatRequest.builder(promptValue)
|
||||
.withTemperature(this.temperature)
|
||||
.withTopP(this.topP)
|
||||
.withTopK(this.topK)
|
||||
.withMaxTokens(this.maxTokens)
|
||||
.withStopSequences(this.stopSequences)
|
||||
.withReturnLikelihoods(this.returnLikelihoods)
|
||||
var request = CohereChatRequest.builder(promptValue)
|
||||
.withTemperature(this.defaultOptions.getTemperature())
|
||||
.withTopP(this.defaultOptions.getTopP())
|
||||
.withTopK(this.defaultOptions.getTopK())
|
||||
.withMaxTokens(this.defaultOptions.getMaxTokens())
|
||||
.withStopSequences(this.defaultOptions.getStopSequences())
|
||||
.withReturnLikelihoods(this.defaultOptions.getReturnLikelihoods())
|
||||
.withStream(stream)
|
||||
.withNumGenerations(this.numGenerations)
|
||||
.withLogitBias(this.logitBias)
|
||||
.withTruncate(this.truncate)
|
||||
.withNumGenerations(this.defaultOptions.getNumGenerations())
|
||||
.withLogitBias(this.defaultOptions.getLogitBias())
|
||||
.withTruncate(this.defaultOptions.getTruncate())
|
||||
.build();
|
||||
|
||||
if (prompt.getOptions() != null) {
|
||||
if (prompt.getOptions() instanceof ChatOptions runtimeOptions) {
|
||||
BedrockCohereChatOptions updatedRuntimeOptions = ModelOptionsUtils.copyToTarget(runtimeOptions,
|
||||
ChatOptions.class, BedrockCohereChatOptions.class);
|
||||
request = ModelOptionsUtils.merge(updatedRuntimeOptions, request, CohereChatRequest.class);
|
||||
}
|
||||
else {
|
||||
throw new IllegalArgumentException("Prompt options are not of type ChatOptions: "
|
||||
+ prompt.getOptions().getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
* Copyright 2024-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.bedrock.cohere;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude.Include;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.LogitBias;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.ReturnLikelihoods;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.Truncate;
|
||||
import org.springframework.ai.chat.ChatOptions;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
* @since 0.8.0
|
||||
*/
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public class BedrockCohereChatOptions implements ChatOptions {
|
||||
|
||||
// @formatter:off
|
||||
/**
|
||||
* (optional) Use a lower value to decrease randomness in the response. Defaults to
|
||||
* 0.7.
|
||||
*/
|
||||
@JsonProperty("temperature") Float temperature;
|
||||
/**
|
||||
* (optional) The maximum cumulative probability of tokens to consider when sampling.
|
||||
* The generative uses combined Top-k and nucleus sampling. Nucleus sampling considers
|
||||
* the smallest set of tokens whose probability sum is at least topP.
|
||||
*/
|
||||
@JsonProperty("p") Float topP;
|
||||
/**
|
||||
* (optional) Specify the number of token choices the generative uses to generate the
|
||||
* next token.
|
||||
*/
|
||||
@JsonProperty("k") Integer topK;
|
||||
/**
|
||||
* (optional) Specify the maximum number of tokens to use in the generated response.
|
||||
*/
|
||||
@JsonProperty("max_tokens") Integer maxTokens;
|
||||
/**
|
||||
* (optional) Configure up to four sequences that the generative recognizes. After a
|
||||
* stop sequence, the generative stops generating further tokens. The returned text
|
||||
* doesn't contain the stop sequence.
|
||||
*/
|
||||
@JsonProperty("stop_sequences") List<String> stopSequences;
|
||||
/**
|
||||
* (optional) Specify how and if the token likelihoods are returned with the response.
|
||||
*/
|
||||
@JsonProperty("return_likelihoods") ReturnLikelihoods returnLikelihoods;
|
||||
/**
|
||||
* (optional) The maximum number of generations that the generative should return.
|
||||
*/
|
||||
@JsonProperty("num_generations") Integer numGenerations;
|
||||
/**
|
||||
* Prevents the model from generating unwanted tokens or incentivize the model to include desired tokens.
|
||||
*/
|
||||
@JsonProperty("logit_bias") LogitBias logitBias;
|
||||
/**
|
||||
* (optional) Specifies how the API handles inputs longer than the maximum token
|
||||
* length.
|
||||
*/
|
||||
@JsonProperty("truncate") Truncate truncate;
|
||||
// @formatter:on
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
|
||||
private final BedrockCohereChatOptions options = new BedrockCohereChatOptions();
|
||||
|
||||
public Builder withTemperature(Float temperature) {
|
||||
this.options.setTemperature(temperature);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withTopP(Float topP) {
|
||||
this.options.setTopP(topP);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withTopK(Integer topK) {
|
||||
this.options.setTopK(topK);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withMaxTokens(Integer maxTokens) {
|
||||
this.options.setMaxTokens(maxTokens);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withStopSequences(List<String> stopSequences) {
|
||||
this.options.setStopSequences(stopSequences);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withReturnLikelihoods(ReturnLikelihoods returnLikelihoods) {
|
||||
this.options.setReturnLikelihoods(returnLikelihoods);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withNumGenerations(Integer numGenerations) {
|
||||
this.options.setNumGenerations(numGenerations);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withLogitBias(LogitBias logitBias) {
|
||||
this.options.setLogitBias(logitBias);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withTruncate(Truncate truncate) {
|
||||
this.options.setTruncate(truncate);
|
||||
return this;
|
||||
}
|
||||
|
||||
public BedrockCohereChatOptions build() {
|
||||
return this.options;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public Float getTemperature() {
|
||||
return this.temperature;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTemperature(Float temperature) {
|
||||
this.temperature = temperature;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Float getTopP() {
|
||||
return this.topP;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTopP(Float topP) {
|
||||
this.topP = topP;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer getTopK() {
|
||||
return this.topK;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTopK(Integer topK) {
|
||||
this.topK = topK;
|
||||
}
|
||||
|
||||
public Integer getMaxTokens() {
|
||||
return this.maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokens(Integer maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
public List<String> getStopSequences() {
|
||||
return this.stopSequences;
|
||||
}
|
||||
|
||||
public void setStopSequences(List<String> stopSequences) {
|
||||
this.stopSequences = stopSequences;
|
||||
}
|
||||
|
||||
public ReturnLikelihoods getReturnLikelihoods() {
|
||||
return this.returnLikelihoods;
|
||||
}
|
||||
|
||||
public void setReturnLikelihoods(ReturnLikelihoods returnLikelihoods) {
|
||||
this.returnLikelihoods = returnLikelihoods;
|
||||
}
|
||||
|
||||
public Integer getNumGenerations() {
|
||||
return this.numGenerations;
|
||||
}
|
||||
|
||||
public void setNumGenerations(Integer numGenerations) {
|
||||
this.numGenerations = numGenerations;
|
||||
}
|
||||
|
||||
public LogitBias getLogitBias() {
|
||||
return this.logitBias;
|
||||
}
|
||||
|
||||
public void setLogitBias(LogitBias logitBias) {
|
||||
this.logitBias = logitBias;
|
||||
}
|
||||
|
||||
public Truncate getTruncate() {
|
||||
return this.truncate;
|
||||
}
|
||||
|
||||
public void setTruncate(Truncate truncate) {
|
||||
this.truncate = truncate;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
/*
|
||||
* Copyright 2024-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.ai.bedrock.cohere;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatModel;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.LogitBias;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.ReturnLikelihoods;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.Truncate;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
public class BedrockCohereChatCreateRequestTests {
|
||||
|
||||
private CohereChatBedrockApi chatApi = new CohereChatBedrockApi(CohereChatModel.COHERE_COMMAND_V14.id(),
|
||||
EnvironmentVariableCredentialsProvider.create(), Region.US_EAST_1.id(), new ObjectMapper());
|
||||
|
||||
@Test
|
||||
public void createRequestWithChatOptions() {
|
||||
|
||||
var client = new BedrockCohereChatClient(chatApi,
|
||||
BedrockCohereChatOptions.builder()
|
||||
.withTemperature(66.6f)
|
||||
.withTopK(66)
|
||||
.withTopP(0.66f)
|
||||
.withMaxTokens(678)
|
||||
.withStopSequences(List.of("stop1", "stop2"))
|
||||
.withReturnLikelihoods(ReturnLikelihoods.ALL)
|
||||
.withNumGenerations(3)
|
||||
.withLogitBias(new LogitBias("t", 6.6f))
|
||||
.withTruncate(Truncate.END)
|
||||
.build());
|
||||
|
||||
CohereChatRequest request = client.createRequest(new Prompt("Test message content"), true);
|
||||
|
||||
assertThat(request.prompt()).isNotEmpty();
|
||||
assertThat(request.stream()).isTrue();
|
||||
|
||||
assertThat(request.temperature()).isEqualTo(66.6f);
|
||||
assertThat(request.topK()).isEqualTo(66);
|
||||
assertThat(request.topP()).isEqualTo(0.66f);
|
||||
assertThat(request.maxTokens()).isEqualTo(678);
|
||||
assertThat(request.stopSequences()).containsExactly("stop1", "stop2");
|
||||
assertThat(request.returnLikelihoods()).isEqualTo(ReturnLikelihoods.ALL);
|
||||
assertThat(request.numGenerations()).isEqualTo(3);
|
||||
assertThat(request.logitBias()).isEqualTo(new LogitBias("t", 6.6f));
|
||||
assertThat(request.truncate()).isEqualTo(Truncate.END);
|
||||
|
||||
request = client.createRequest(new Prompt("Test message content",
|
||||
BedrockCohereChatOptions.builder()
|
||||
.withTemperature(99.9f)
|
||||
.withTopK(99)
|
||||
.withTopP(0.99f)
|
||||
.withMaxTokens(888)
|
||||
.withStopSequences(List.of("stop3", "stop4"))
|
||||
.withReturnLikelihoods(ReturnLikelihoods.GENERATION)
|
||||
.withNumGenerations(13)
|
||||
.withLogitBias(new LogitBias("t", 9.9f))
|
||||
.withTruncate(Truncate.START)
|
||||
.build()),
|
||||
false
|
||||
|
||||
);
|
||||
|
||||
assertThat(request.prompt()).isNotEmpty();
|
||||
assertThat(request.stream()).isFalse();
|
||||
|
||||
assertThat(request.temperature()).isEqualTo(99.9f);
|
||||
assertThat(request.topK()).isEqualTo(99);
|
||||
assertThat(request.topP()).isEqualTo(0.99f);
|
||||
assertThat(request.maxTokens()).isEqualTo(888);
|
||||
assertThat(request.stopSequences()).containsExactly("stop3", "stop4");
|
||||
assertThat(request.returnLikelihoods()).isEqualTo(ReturnLikelihoods.GENERATION);
|
||||
assertThat(request.numGenerations()).isEqualTo(13);
|
||||
assertThat(request.logitBias()).isEqualTo(new LogitBias("t", 9.9f));
|
||||
assertThat(request.truncate()).isEqualTo(Truncate.START);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,128 +1,3 @@
|
||||
# 1. Ollama Chat and Embedding
|
||||
# Ollama Chat
|
||||
|
||||
## 1.1 OllamaApi
|
||||
|
||||
[OllamaApi](./src/main/java/org/springframework/ai/ollama/api/OllamaApi.java) provides is lightweight Java client for [Ollama models](https://ollama.ai/).
|
||||
|
||||
The OllamaApi provides the Chat completion as well as Embedding endpoints.
|
||||
|
||||
Following class diagram illustrates the OllamaApi interface and building blocks for chat completion:
|
||||
|
||||
|
||||
|
||||
The OllamaApi can supports all [Ollama Models](https://ollama.ai/library) providing synchronous chat completion, streaming chat completion and embedding:
|
||||
|
||||
```java
|
||||
ChatResponse chat(ChatRequest chatRequest)
|
||||
|
||||
Flux<ChatResponse> streamingChat(ChatRequest chatRequest)
|
||||
|
||||
EmbeddingResponse embeddings(EmbeddingRequest embeddingRequest)
|
||||
```
|
||||
|
||||
> NOTE: OllamaApi expose also the Ollama `generation` endpoint but later is inferior compared to the Ollama `chat` endpoint.
|
||||
|
||||
The `OllamaOptions` is helper class used as type-safe option builder.
|
||||
It provides `toMap` to convert the content into `Map<String, Object>`.
|
||||
|
||||
Here is a simple snippet how to use the OllamaApi programmatically:
|
||||
|
||||
```java
|
||||
var request = ChatRequest.builder("orca-mini")
|
||||
.withStream(false)
|
||||
.withMessages(List.of(Message.builder(Role.user)
|
||||
.withContent("What is the capital of Bulgaria and what is the size? " + "What it the national anthem?")
|
||||
.build()))
|
||||
.withOptions(Options.create().withTemperature(0.9f).withTopK(10))
|
||||
.build();
|
||||
|
||||
ChatResponse response = ollamaApi.chat(request);
|
||||
```
|
||||
|
||||
```java
|
||||
var request = ChatRequest.builder("orca-mini")
|
||||
.withStream(true)
|
||||
.withMessages(List.of(Message.builder(Role.user)
|
||||
.withContent("What is the capital of Bulgaria and what is the size? " + "What it the national anthem?")
|
||||
.build()))
|
||||
.withOptions(Options.create().withTemperature(0.9f))
|
||||
.build();
|
||||
|
||||
Flux<ChatResponse> response = ollamaApi.streamingChat(request);
|
||||
|
||||
List<ChatResponse> responses = response.collectList().block();
|
||||
```
|
||||
|
||||
```java
|
||||
EmbeddingRequest request = new EmbeddingRequest("orca-mini", "I like to eat apples");
|
||||
|
||||
EmbeddingResponse response = ollamaApi.embeddings(request);
|
||||
```
|
||||
|
||||
## 1.2 OllamaChatClient and OllamaEmbeddingClient
|
||||
|
||||
The [OllamaChatClient](./src/main/java/org/springframework/ai/ollama/OllamaChatClient.java) implements the Spring-Ai `ChatClient` and `StreamingChatClient` interfaces.
|
||||
|
||||
The [OllamaEmbeddingClient](./src/main/java/org/springframework/ai/ollama/OllamaEmbeddingClient.java) implements the Spring AI `EmbeddingClient` interface.
|
||||
|
||||
Both the OllamaChatClient and the OllamaEmbeddingClient leverage the `OllamaApi`.
|
||||
|
||||
You can configure the clients like this:
|
||||
|
||||
```java
|
||||
@Bean
|
||||
public OllamaApi ollamaApi() {
|
||||
return new OllamaApi(baseUrl);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public OllamaChatClient ollamaChat(OllamaApi ollamaApi) {
|
||||
return new OllamaChatClient(ollamaApi)
|
||||
.withModel("llama2")
|
||||
.withOptions(OllamaOptions.create()
|
||||
.withTemperature(0.9f)
|
||||
.withTopK(12));
|
||||
}
|
||||
|
||||
@Bean
|
||||
public OllamaEmbeddingClient ollamaEmbedding(OllamaApi ollamaApi) {
|
||||
return new OllamaEmbeddingClient(ollamaApi)
|
||||
.withModel("orca-mini");
|
||||
}
|
||||
```
|
||||
|
||||
or you can leverage the `spring-ai-ollama-spring-boot-starter` Spring Boot starter.
|
||||
For this add the following dependency:
|
||||
|
||||
```xml
|
||||
<dependency>
|
||||
<artifactId>spring-ai-ollama-spring-boot-starter</artifactId>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
Use the `OllamaConnectionProperties` to configure the Ollama clients (both Chat and Embedding) connections:
|
||||
|
||||
| Property | Description | Default |
|
||||
| ------------- | ------------- | ------------- |
|
||||
| spring.ai.ollama.base-url | The base url of the Ollama server. | http://localhost:11434 |
|
||||
|
||||
Use the `OllamaChatProperties` to configure the Ollama Chat client:
|
||||
|
||||
| Property | Description | Default |
|
||||
| ------------- | ------------- | ------------- |
|
||||
| spring.ai.ollama.chat.model | Model to use. | llama2 |
|
||||
| spring.ai.ollama.chat.enabled | Allows you to disable the Ollama Chat autoconfiguration. | true |
|
||||
| spring.ai.ollama.chat.temperature | Controls the randomness of the output. Values can range over [0.0,1.0] | 0.8 |
|
||||
| spring.ai.ollama.chat.topP | The maximum cumulative probability of tokens to consider when sampling. | - |
|
||||
| spring.ai.ollama.chat.topK | Max number or responses to generate. | - |
|
||||
| spring.ai.options.chat.options | A `OllamaOptions` used to configure the Chat client. | - |
|
||||
|
||||
and `OllamaEmbeddingProperties` to configure the Ollama Embedding client:
|
||||
|
||||
| Property | Description | Default |
|
||||
| ------------- | ------------- | ------------- |
|
||||
| spring.ai.ollama.embedding.model | Model to use. | llama2 |
|
||||
| spring.ai.ollama.embedding.enabled | Allows you to disable the Ollama embedding autoconfiguration. | true |
|
||||
| spring.ai.options.embedding.options | `OllamaOptions` used to configure the embedding client. | - |
|
||||
Visit the Spring AI [Ollama Chat Documentation](https://docs.spring.io/spring-ai/reference/api/clients/ollama-chat.html).
|
||||
|
||||
3
models/spring-ai-openai/README.md
Normal file
3
models/spring-ai-openai/README.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# OpenAI Chat
|
||||
|
||||
Visit the Spring AI [OpenAI Chat Documentation](https://docs.spring.io/spring-ai/reference/api/clients/openai-chat.html).
|
||||
@@ -64,10 +64,7 @@ public class OpenAiChatClient implements ChatClient, StreamingChatClient {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(getClass());
|
||||
|
||||
private OpenAiChatOptions defaultOptions = OpenAiChatOptions.builder()
|
||||
.withModel("gpt-3.5-turbo")
|
||||
.withTemperature(0.7f)
|
||||
.build();
|
||||
private OpenAiChatOptions defaultOptions;
|
||||
|
||||
public final RetryTemplate retryTemplate = RetryTemplate.builder()
|
||||
.maxAttempts(10)
|
||||
@@ -84,10 +81,21 @@ public class OpenAiChatClient implements ChatClient, StreamingChatClient {
|
||||
private final OpenAiApi openAiApi;
|
||||
|
||||
public OpenAiChatClient(OpenAiApi openAiApi) {
|
||||
Assert.notNull(openAiApi, "OpenAiApi must not be null");
|
||||
this.openAiApi = openAiApi;
|
||||
this(openAiApi, OpenAiChatOptions.builder().withModel("gpt-3.5-turbo").withTemperature(0.7f).build());
|
||||
}
|
||||
|
||||
public OpenAiChatClient(OpenAiApi openAiApi, OpenAiChatOptions options) {
|
||||
Assert.notNull(openAiApi, "OpenAiApi must not be null");
|
||||
Assert.notNull(options, "Options must not be null");
|
||||
this.openAiApi = openAiApi;
|
||||
this.defaultOptions = options;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated since 0.8.0, use the
|
||||
* {@link #OpenAiChatClient(OpenAiApi, OpenAiChatOptions)} constructor instead.
|
||||
*/
|
||||
@Deprecated(since = "0.8.0", forRemoval = true)
|
||||
public OpenAiChatClient withDefaultOptions(OpenAiChatOptions options) {
|
||||
this.defaultOptions = options;
|
||||
return this;
|
||||
|
||||
@@ -31,8 +31,8 @@ public class ChatCompletionRequestTests {
|
||||
@Test
|
||||
public void createRequestWithChatOptions() {
|
||||
|
||||
var client = new OpenAiChatClient(new OpenAiApi("TEST"))
|
||||
.withDefaultOptions(OpenAiChatOptions.builder().withModel("DEFAULT_MODEL").withTemperature(66.6f).build());
|
||||
var client = new OpenAiChatClient(new OpenAiApi("TEST"),
|
||||
OpenAiChatOptions.builder().withModel("DEFAULT_MODEL").withTemperature(66.6f).build());
|
||||
|
||||
var request = client.createRequest(new Prompt("Test message content"), false);
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 164 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 194 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 424 KiB |
@@ -8,6 +8,7 @@
|
||||
*** xref:api/clients/bedrock.adoc[]
|
||||
**** xref:api/clients/bedrock/bedrock-anthropic.adoc[]
|
||||
**** xref:api/clients/bedrock/bedrock-llama2.adoc[]
|
||||
**** xref:api/clients/bedrock/bedrock-cohere.adoc[]
|
||||
*** xref:api/clients/huggingface.adoc[]
|
||||
*** xref:api/clients/ollama-chat.adoc[]
|
||||
** xref:api/prompt.adoc[]
|
||||
|
||||
@@ -7,10 +7,8 @@ Azure offers Java developers the opportunity to leverage AI's full potential by
|
||||
== Prerequisites
|
||||
|
||||
Obtain your Azure OpenAI `endpoint` and `api-key` from the Azure OpenAI Service section on the link:https://portal.azure.com[Azure Portal].
|
||||
|
||||
Spring AI defines a configuration property named `spring.ai.azure.openai.api-key` that you should set to the value of the `API Key` obtained from Azure.
|
||||
There is also a configuration property named `spring.ai.azure.openai.endpoint` that you should set to the endpoint URL obtained when provisioning your model in Azure.
|
||||
|
||||
Exporting environment variables is one way to set these configuration properties:
|
||||
|
||||
[source,shell]
|
||||
@@ -42,6 +40,8 @@ dependencies {
|
||||
}
|
||||
----
|
||||
|
||||
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
|
||||
=== Chat Properties
|
||||
|
||||
The prefix `spring.ai.azure.openai` is the property prefix to configure the connection to Azure OpenAI.
|
||||
@@ -73,10 +73,36 @@ The prefix `spring.ai.azure.openai.chat` is the property prefix that configures
|
||||
| spring.ai.azure.openai.chat.options.frequencyPenalty | A value that influences the probability of generated tokens appearing based on their cumulative frequency in generated text. Positive values will make tokens less likely to appear as their frequency increases and decrease the likelihood of the model repeating the same statements verbatim. | -
|
||||
|====
|
||||
|
||||
=== Sample Code
|
||||
TIP: All properties prefixed with `spring.ai.azure.openai.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.
|
||||
|
||||
This will create a `ChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the `ChatClient` implementation.
|
||||
=== Chat Options [[chat-options]]
|
||||
|
||||
The link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatOptions.java[AzureOpenAiChatOptions.java] provides model configurations, such as the model to use, the temperature, the frequency penalty, etc.
|
||||
|
||||
On start-up, the default options can be configured with the `AzureOpenAiChatClient(api, options)` constructor or the `spring.ai.azure.openai.chat.options.*` properties.
|
||||
|
||||
At run-time you can override the default options by adding new, request specific, options to the `Prompt` call.
|
||||
For example to override the default model and temperature for a specific request:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
AzureOpenAiChatOptions.builder()
|
||||
.withModel("gpt-4-32k")
|
||||
.withTemperature(0.4)
|
||||
.build()
|
||||
));
|
||||
----
|
||||
|
||||
TIP: In addition to the model specific link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatOptions.java[AzureOpenAiChatOptions.java] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptions.java[ChatOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptionsBuilder.java[ChatOptionsBuilder#builder()].
|
||||
|
||||
=== Sample Controller (Auto-configuration)
|
||||
|
||||
https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-azure-openai-spring-boot-starter` to your pom (or gradle) dependencies.
|
||||
|
||||
Add a `application.properties` file, under the `src/main/resources` directory, to enable and configure the OpenAi Chat client:
|
||||
|
||||
[source,application.properties]
|
||||
----
|
||||
@@ -86,15 +112,21 @@ spring.ai.azure.openai.chat.options.model=gpt-35-turbo
|
||||
spring.ai.azure.openai.chat.options.temperature=0.7
|
||||
----
|
||||
|
||||
TIP: replace the `api-key` and `endpoint` with your Azure OpenAI credentials.
|
||||
|
||||
This will create a `AzureOpenAiChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the chat client for text generations.
|
||||
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@RestController
|
||||
public class ChatController {
|
||||
|
||||
private final ChatClient chatClient;
|
||||
private final AzureOpenAiChatClient chatClient;
|
||||
|
||||
@Autowired
|
||||
public ChatController(ChatClient chatClient) {
|
||||
public ChatController(AzureOpenAiChatClient chatClient) {
|
||||
this.chatClient = chatClient;
|
||||
}
|
||||
|
||||
@@ -102,12 +134,20 @@ public class ChatController {
|
||||
public Map generate(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
return Map.of("generation", chatClient.generate(message));
|
||||
}
|
||||
|
||||
@GetMapping("/open-ai/generateStream")
|
||||
public Flux<ChatResponse> generateStream(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
Prompt prompt = new Prompt(new UserMessage(message));
|
||||
return chatClient.stream(prompt);
|
||||
}
|
||||
}
|
||||
----
|
||||
|
||||
== Manual Configuration
|
||||
|
||||
Add the `spring-ai-azure-openai` dependency to your project's Maven `pom.xml` file:
|
||||
The link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatClient.java[AzureOpenAiChatClient] implements the `ChatClient` and `StreamingChatClient` and uses the link:https://learn.microsoft.com/en-us/java/api/overview/azure/ai-openai-readme?view=azure-java-preview[Azure OpenAI Java Client].
|
||||
|
||||
To enable it, add the `spring-ai-azure-openai` dependency to your project's Maven `pom.xml` file:
|
||||
[source, xml]
|
||||
----
|
||||
<dependency>
|
||||
@@ -126,7 +166,7 @@ dependencies {
|
||||
}
|
||||
----
|
||||
|
||||
NOTE: The `spring-ai-azure-openai` dependency also provide the access to the `AzureOpenAiChatClient`. For more information about the `AzureOpenAiChatClient` refer to the link:../clients/azure-openai-chat.html[Azure OpenAI Chat] section.
|
||||
TIP: The `spring-ai-azure-openai` dependency also provide the access to the `AzureOpenAiChatClient`. For more information about the `AzureOpenAiChatClient` refer to the link:../clients/azure-openai-chat.html[Azure OpenAI Chat] section.
|
||||
|
||||
Next, create an `AzureOpenAiChatClient` instance and use it to generate text responses:
|
||||
|
||||
@@ -155,21 +195,3 @@ Flux<ChatResponse> response = chatClient.stream(
|
||||
|
||||
NOTE: the `gpt-35-turbo` is actually the `Deployment Name` as presented in the Azure AI Portal.
|
||||
|
||||
=== Chat Options
|
||||
|
||||
The `AzureOpenAiChatOptions` provides the configuration information for the chat requests.
|
||||
The `AzureOpenAiChatOptions` offers a builder to create the options.
|
||||
|
||||
At start time use the `AzureOpenAiChatClient` constructor to set the default options used for all char requests.
|
||||
At runtime, you can override the default options by passing a `AzureOpenAiChatOptions` instance with your to the `Prompt` request.
|
||||
|
||||
For example to override the default model name for a specific request:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
AzureOpenAiChatOptions.builder().withModel("gpt-4-32k").build()
|
||||
));
|
||||
----
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
= Amazon Bedrock
|
||||
= Amazon Bedrock Chat
|
||||
|
||||
link:https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html[Amazon Bedrock] is a managed service that provides foundation models from various AI providers, available through a unified API.
|
||||
|
||||
@@ -86,6 +86,7 @@ For more information, refer to the documentation below for each supported model.
|
||||
|
||||
* xref:api/clients/bedrock/bedrock-anthropic.adoc[Spring AI Bedrock Anthropic Chat]: `spring.ai.bedrock.anthropic.chat.enabled=true`
|
||||
* xref:api/clients/bedrock/bedrock-llama2.adoc[Spring AI Bedrock Llama2 Chat]: `spring.ai.bedrock.llama2.chat.enabled=true`
|
||||
* xref:api/clients/bedrock/bedrock-cohere.adoc[Spring AI Bedrock Cohere Chat]: `spring.ai.bedrock.cohere.chat.enabled=true`
|
||||
|
||||
|
||||
// * [Spring AI Bedrock Cohere Chat](./README_COHERE_CHAT.md) - `spring.ai.bedrock.cohere.chat.enabled=true`
|
||||
|
||||
@@ -1,22 +1,21 @@
|
||||
= Anthropic Chat
|
||||
|
||||
https://www.anthropic.com/product[Anthropic's Claude] is an AI assistant based on Anthropic’s research into training helpful, honest, and harmless AI systems.
|
||||
|
||||
The Claude model has the following high level features
|
||||
|
||||
* 200k Token Context Window: Claude boasts a generous token capacity of 200,000, making it ideal for handling extensive information in applications like technical documentation, codebases, and literary works.
|
||||
* 200k Token Context Window: Claude boasts a generous token capacity of 200,000, making it ideal for handling extensive information in applications like technical documentation, codebase, and literary works.
|
||||
* Supported Tasks: Claude's versatility spans tasks such as summarization, Q&A, trend forecasting, and document comparisons, enabling a wide range of applications from dialogues to content generation.
|
||||
* AI Safety Features: Built on Anthropic's safety research, Claude prioritizes helpfulness, honesty, and harmlessness in its interactions, reducing brand risk and ensuring responsible AI behavior.
|
||||
|
||||
The https://aws.amazon.com/bedrock/claude[AWS Bedrock Anthropic Model Page] and https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html[Amazon Bedrock User Guide] contains detailed information on how to use the AWS hosted model.
|
||||
|
||||
== Pre-requisites
|
||||
== Prerequisites
|
||||
|
||||
Refer to the xref:api/clients/bedrock.adoc[Spring AI documentation on Amazon Bedrock] for setting up API access.
|
||||
|
||||
== Auto-configuration
|
||||
|
||||
or you can leverage the `spring-ai-bedrock-ai-spring-boot-starter` Spring Boot starter:
|
||||
Add the `spring-ai-bedrock-ai-spring-boot-starter` dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
@@ -27,10 +26,20 @@ or you can leverage the `spring-ai-bedrock-ai-spring-boot-starter` Spring Boot s
|
||||
</dependency>
|
||||
----
|
||||
|
||||
=== Enable Anthropic Support
|
||||
or to your Gradle `build.gradle` build file.
|
||||
|
||||
Spring AI defines a configuration property named `spring.ai.bedrock.anthropic.chat.enabled` that you should set to `true` to enable support for Anthropic.
|
||||
Exporting environment variables in one way to set this configuration property.
|
||||
[source,gradle]
|
||||
----
|
||||
dependencies {
|
||||
implementation 'org.springframework.ai:spring-ai-bedrock-ai-spring-boot-starter:0.8.0-SNAPSHOT'
|
||||
}
|
||||
----
|
||||
|
||||
=== Enable Anthropic Chat
|
||||
|
||||
By default the Anthropic model is disabled.
|
||||
To enable it set the `spring.ai.bedrock.anthropic.chat.enabled` property to `true`.
|
||||
Exporting environment variable is one way to set this configuration property:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
@@ -45,19 +54,19 @@ The prefix `spring.ai.bedrock.aws` is the property prefix to configure the conne
|
||||
|====
|
||||
| Property | Description | Default
|
||||
|
||||
| spring.ai.bedrock.aws.region | AWS region to use. | us-east-1
|
||||
| spring.ai.bedrock.aws.region | AWS region to use. | us-east-1
|
||||
| spring.ai.bedrock.aws.access-key | AWS access key. | -
|
||||
| spring.ai.bedrock.aws.secret-key | AWS secret key. | -
|
||||
|====
|
||||
|
||||
The prefix `spring.ai.bedrock.anthropic.chat` is the property prefix that configures the `ChatClient` implementation for Claude.
|
||||
The prefix `spring.ai.bedrock.anthropic.chat` is the property prefix that configures the chat client implementation for Claude.
|
||||
|
||||
[cols="2,5,1"]
|
||||
|====
|
||||
| Property | Description | Default
|
||||
|
||||
| spring.ai.bedrock.anthropic.chat.enable | Enable Bedrock Anthropic chat client. Disabled by default | false
|
||||
| spring.ai.bedrock.anthropic.chat.model | The model id to use. See the `AnthropicChatModel` for the supported models. | anthropic.claude-v2
|
||||
| spring.ai.bedrock.anthropic.chat.model | The model id to use. See the https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/api/AnthropicChatBedrockApi.java[AnthropicChatModel] for the supported models. | anthropic.claude-v2
|
||||
| spring.ai.bedrock.anthropic.chat.options.temperature | Controls the randomness of the output. Values can range over [0.0,1.0] | 0.8
|
||||
| spring.ai.bedrock.anthropic.chat.options.topP | The maximum cumulative probability of tokens to consider when sampling. | AWS Bedrock default
|
||||
| spring.ai.bedrock.anthropic.chat.options.topK | Specify the number of token choices the generative uses to generate the next token. | AWS Bedrock default
|
||||
@@ -66,46 +75,84 @@ The prefix `spring.ai.bedrock.anthropic.chat` is the property prefix that config
|
||||
| spring.ai.bedrock.anthropic.chat.options.maxTokensToSample | Specify the maximum number of tokens to use in the generated response. Note that the models may stop before reaching this maximum. This parameter only specifies the absolute maximum number of tokens to generate. We recommend a limit of 4,000 tokens for optimal performance. | 500
|
||||
|====
|
||||
|
||||
Look at the Spring AI enumeration `AnthropicChatModel` for other model IDs. The other value supported is `anthropic.claude-instant-v1`.
|
||||
|
||||
Look at the https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/api/AnthropicChatBedrockApi.java[AnthropicChatModel] for other model IDs.
|
||||
Supported values are: `anthropic.claude-instant-v1`, `anthropic.claude-v2` and `anthropic.claude-v2:1`.
|
||||
Model ID values can also be found in the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html[AWS Bedrock documentation for base model IDs].
|
||||
|
||||
=== Sample Code
|
||||
TIP: All properties prefixed with `spring.ai.bedrock.anthropic.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.
|
||||
|
||||
This will create a `ChatClient` implementation that you can inject into your class.
|
||||
=== Chat Options [[chat-options]]
|
||||
|
||||
Create an `application.properties` file in the `src/main/resources` directory and add the following properties to configure the Anthropic Chat client.
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/AnthropicChatOptions.java[AnthropicChatOptions.java] provides model configurations, such as temperature, topK, topP, etc.
|
||||
|
||||
On start-up, the default options can be configured with the `BedrockAnthropicChatClient(api, options)` constructor or the `spring.ai.bedrock.anthropic.chat.options.*` properties.
|
||||
|
||||
At run-time you can override the default options by adding new, request specific, options to the `Prompt` call.
|
||||
For example to override the default temperature for a specific request:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
AnthropicChatOptions.builder()
|
||||
.withTemperature(0.4)
|
||||
.build()
|
||||
));
|
||||
----
|
||||
|
||||
TIP: In addition to the model specific https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/AnthropicChatOptions.java[AnthropicChatOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptions.java[ChatOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptionsBuilder.java[ChatOptionsBuilder#builder()].
|
||||
|
||||
=== Sample Controller (Auto-configuration)
|
||||
|
||||
https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-bedrock-ai-spring-boot-starter` to your pom (or gradle) dependencies.
|
||||
|
||||
Add a `application.properties` file, under the `src/main/resources` directory, to enable and configure the Anthropic Chat client:
|
||||
|
||||
[source]
|
||||
----
|
||||
spring.ai.bedrock.aws.region=eu-central-1
|
||||
spring.ai.bedrock.aws.access-key=${AWS_ACCESS_KEY_ID}
|
||||
spring.ai.bedrock.aws.secret-key=${AWS_SECRET_ACCESS_KEY}
|
||||
|
||||
spring.ai.bedrock.anthropic.chat.enabled=true
|
||||
spring.ai.bedrock.anthropic.chat.options.temperature=0.8
|
||||
spring.ai.bedrock.anthropic.chat.options.top-k=15
|
||||
----
|
||||
|
||||
Here is an example of a simple `@Controller` class that uses the `ChatClient` implementation.
|
||||
TIP: replace the `regions`, `access-key` and `secret-key` with your AWS credentials.
|
||||
|
||||
This will create a `BedrockAnthropicChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the chat client for text generations.
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@RestController
|
||||
public class ChatController {
|
||||
|
||||
private final ChatClient chatClient;
|
||||
private final BedrockAnthropicChatClient chatClient;
|
||||
|
||||
@Autowired
|
||||
public ChatController(ChatClient chatClient) {
|
||||
public ChatController(BedrockAnthropicChatClient chatClient) {
|
||||
this.chatClient = chatClient;
|
||||
}
|
||||
|
||||
@GetMapping("/ai/generate")
|
||||
public Map generate(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
return Map.of("generation", chatClient.generate(message));
|
||||
return Map.of("generation", chatClient.call(message));
|
||||
}
|
||||
|
||||
@GetMapping("/open-ai/generateStream")
|
||||
public Flux<ChatResponse> generateStream(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
Prompt prompt = new Prompt(new UserMessage(message));
|
||||
return chatClient.stream(prompt);
|
||||
}
|
||||
}
|
||||
----
|
||||
|
||||
== Manual Configuration
|
||||
|
||||
The link:./src/main/java/org/springframework/ai/bedrock/anthropic/BedrockAnthropicChatClient.java[BedrockAnthropicChatClient] implements the `ChatClient` and `StreamingChatClient` and uses the `AnthropicChatBedrockApi` library to connect to the Bedrock Anthropic service.
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/BedrockAnthropicChatClient.java[BedrockAnthropicChatClient] implements the `ChatClient` and `StreamingChatClient` and uses the <<low-level-api>> to connect to the Bedrock Anthropic service.
|
||||
|
||||
Add the `spring-ai-bedrock` dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
@@ -127,9 +174,9 @@ dependencies {
|
||||
}
|
||||
----
|
||||
|
||||
NOTE: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
|
||||
Next, create an `BedrockAnthropicChatClient` instance and use it to text generations requests:
|
||||
Next, create an https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/BedrockAnthropicChatClient.java[BedrockAnthropicChatClient] and use it for text generations:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@@ -156,47 +203,36 @@ Flux<ChatResponse> response = chatClient.stream(
|
||||
new Prompt("Generate the names of 5 famous pirates."));
|
||||
----
|
||||
|
||||
=== Low-level AnthropicChatBedrockApi Client [[low-level-api]]
|
||||
|
||||
== Appendices
|
||||
|
||||
=== Using low-level AnthropicChatBedrockApi Library
|
||||
|
||||
The link:./src/main/java/org/springframework/ai/bedrock/anthropic/api/AnthropicChatBedrockApi.java[AnthropicChatBedrockApi] provides is lightweight Java client on top of AWS Bedrock link:https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-claude.html[Anthropic Claude models].
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/api/AnthropicChatBedrockApi.java[AnthropicChatBedrockApi] provides is lightweight Java client on top of AWS Bedrock link:https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-claude.html[Anthropic Claude models].
|
||||
|
||||
Following class diagram illustrates the AnthropicChatBedrockApi interface and building blocks:
|
||||
|
||||
image::bedrock/bedrock-anthropic-chat-api.png[AnthropicChatBedrockApi Class Diagram]
|
||||
|
||||
The AnthropicChatBedrockApi supports the `anthropic.claude-instant-v1` and `anthropic.claude-v2` models.
|
||||
|
||||
Also the AnthropicChatBedrockApi supports both synchronous (e.g. `chatCompletion()`) and streaming (e.g. `chatCompletionStream()`) responses.
|
||||
Client supports the `anthropic.claude-instant-v1`, `anthropic.claude-v2` and `anthropic.claude-v2:1` models for both synchronous (e.g. `chatCompletion()`) and streaming (e.g. `chatCompletionStream()`) responses.
|
||||
|
||||
Here is a simple snippet how to use the api programmatically:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
AnthropicChatBedrockApi anthropicChatApi = new AnthropicChatBedrockApi(
|
||||
AnthropicModel.CLAUDE_V2.id(),
|
||||
Region.EU_CENTRAL_1.id());
|
||||
AnthropicModel.CLAUDE_V2.id(), Region.EU_CENTRAL_1.id());
|
||||
|
||||
AnthropicChatRequest request = AnthropicChatRequest
|
||||
.builder(String.format(AnthropicChatBedrockApi.PROMPT_TEMPLATE, "Name 3 famous pirates"))
|
||||
.withTemperature(0.8f)
|
||||
.withMaxTokensToSample(300)
|
||||
.withTopK(10)
|
||||
// .withStopSequences(List.of("\n\nHuman:"))
|
||||
.build();
|
||||
|
||||
// Sync request
|
||||
AnthropicChatResponse response = anthropicChatApi.chatCompletion(request);
|
||||
|
||||
System.out.println(response.completion());
|
||||
|
||||
// Streaming response
|
||||
// Streaming request
|
||||
Flux<AnthropicChatResponse> responseStream = anthropicChatApi.chatCompletionStream(request);
|
||||
|
||||
List<AnthropicChatResponse> responses = responseStream.collectList().block();
|
||||
|
||||
System.out.println(responses);
|
||||
----
|
||||
|
||||
Follow the link:./src/main/java/org/springframework/ai/bedrock/anthropic/api/AnthropicChatBedrockApi.java[AnthropicChatBedrockApi.java]'s JavaDoc for further information.
|
||||
Follow the https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/anthropic/api/AnthropicChatBedrockApi.java[AnthropicChatBedrockApi.java]'s JavaDoc for further information.
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
= Cohere Chat
|
||||
|
||||
Provides Bedrock Cohere Chat client.
|
||||
Integrate generative AI capabilities into essential apps and workflows that improve business outcomes.
|
||||
|
||||
The https://aws.amazon.com/bedrock/cohere-command-embed/[AWS Bedrock Cohere Model Page] and https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html[Amazon Bedrock User Guide] contains detailed information on how to use the AWS hosted model.
|
||||
|
||||
== Prerequisites
|
||||
|
||||
Refer to the xref:api/clients/bedrock.adoc[Spring AI documentation on Amazon Bedrock] for setting up API access.
|
||||
|
||||
== Auto-configuration
|
||||
|
||||
Add the `spring-ai-bedrock-ai-spring-boot-starter` dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-bedrock-ai-spring-boot-starter</artifactId>
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
----
|
||||
|
||||
or to your Gradle `build.gradle` build file.
|
||||
|
||||
[source,gradle]
|
||||
----
|
||||
dependencies {
|
||||
implementation 'org.springframework.ai:spring-ai-bedrock-ai-spring-boot-starter:0.8.0-SNAPSHOT'
|
||||
}
|
||||
----
|
||||
|
||||
=== Enable Cohere Chat Support
|
||||
|
||||
By default the Cohere model is disabled.
|
||||
To enable it set the `spring.ai.bedrock.cohere.chat.enabled` property to `true`.
|
||||
Exporting environment variable is one way to set this configuration property:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
export SPRING_AI_BEDROCK_COHERE_CHAT_ENABLED=true
|
||||
----
|
||||
|
||||
=== Chat Properties
|
||||
|
||||
The prefix `spring.ai.bedrock.aws` is the property prefix to configure the connection to AWS Bedrock.
|
||||
|
||||
[cols="3,3,3"]
|
||||
|====
|
||||
| Property | Description | Default
|
||||
|
||||
| spring.ai.bedrock.aws.region | AWS region to use. | us-east-1
|
||||
| spring.ai.bedrock.aws.access-key | AWS access key. | -
|
||||
| spring.ai.bedrock.aws.secret-key | AWS secret key. | -
|
||||
|====
|
||||
|
||||
The prefix `spring.ai.bedrock.cohere.chat` is the property prefix that configures the chat client implementation for Cohere.
|
||||
|
||||
[cols="2,5,1"]
|
||||
|====
|
||||
| Property | Description | Default
|
||||
|
||||
| spring.ai.bedrock.cohere.chat.enabled | Enable or disable support for Cohere | false
|
||||
| spring.ai.bedrock.cohere.chat.model | The model id to use. See the https://github.com/spring-projects/spring-ai/blob/4ba9a3cd689b9fd3a3805f540debe398a079c6ef/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/cohere/api/CohereChatBedrockApi.java#L326C14-L326C29[CohereChatModel] for the supported models. | cohere.command-text-v14
|
||||
| spring.ai.bedrock.cohere.chat.options.temperature | Controls the randomness of the output. Values can range over [0.0,1.0] | 0.7
|
||||
| spring.ai.bedrock.cohere.chat.options.topP | The maximum cumulative probability of tokens to consider when sampling. | AWS Bedrock default
|
||||
| spring.ai.bedrock.cohere.chat.options.topK | Specify the number of token choices the model uses to generate the next token | AWS Bedrock default
|
||||
| spring.ai.bedrock.cohere.chat.options.maxTokens | Specify the maximum number of tokens to use in the generated response. | AWS Bedrock default
|
||||
| spring.ai.bedrock.cohere.chat.options.stopSequences | Configure up to four sequences that the model recognizes. | AWS Bedrock default
|
||||
| spring.ai.bedrock.cohere.chat.options.returnLikelihoods | The token likelihoods are returned with the response. | AWS Bedrock default
|
||||
| spring.ai.bedrock.cohere.chat.options.numGenerations | The maximum number of generations that the model should return. | AWS Bedrock default
|
||||
| spring.ai.bedrock.cohere.chat.options.logitBias | Prevents the model from generating unwanted tokens or incentivize the model to include desired tokens. | AWS Bedrock default
|
||||
| spring.ai.bedrock.cohere.chat.options.truncate | Specifies how the API handles inputs longer than the maximum token length | AWS Bedrock default
|
||||
|====
|
||||
|
||||
Look at the https://github.com/spring-projects/spring-ai/blob/4ba9a3cd689b9fd3a3805f540debe398a079c6ef/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/cohere/api/CohereChatBedrockApi.java#L326C14-L326C29[CohereChatModel] for other model IDs.
|
||||
Supported values are: `cohere.command-light-text-v14` and `cohere.command-text-v14`.
|
||||
Model ID values can also be found in the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html[AWS Bedrock documentation for base model IDs].
|
||||
|
||||
TIP: All properties prefixed with `spring.ai.bedrock.cohere.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.
|
||||
|
||||
=== Chat Options [[chat-options]]
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/cohere/BedrockCohereChatOptions.java[BedrockCohereChatOptions.java] provides model configurations, such as temperature, topK, topP, etc.
|
||||
|
||||
On start-up, the default options can be configured with the `BedrockCohereChatClient(api, options)` constructor or the `spring.ai.bedrock.cohere.chat.options.*` properties.
|
||||
|
||||
At run-time you can override the default options by adding new, request specific, options to the `Prompt` call.
|
||||
For example to override the default temperature for a specific request:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
BedrockCohereChatOptions.builder()
|
||||
.withTemperature(0.4)
|
||||
.build()
|
||||
));
|
||||
----
|
||||
|
||||
TIP: In addition to the model specific https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/cohere/BedrockCohereChatOptions.java[BedrockCohereChatOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptions.java[ChatOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptionsBuilder.java[ChatOptionsBuilder#builder()].
|
||||
|
||||
=== Sample Controller (Auto-configuration)
|
||||
|
||||
https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-bedrock-ai-spring-boot-starter` to your pom (or gradle) dependencies.
|
||||
|
||||
Add a `application.properties` file, under the `src/main/resources` directory, to enable and configure the Anthropic Chat client:
|
||||
|
||||
[source]
|
||||
----
|
||||
spring.ai.bedrock.aws.region=eu-central-1
|
||||
spring.ai.bedrock.aws.access-key=${AWS_ACCESS_KEY_ID}
|
||||
spring.ai.bedrock.aws.secret-key=${AWS_SECRET_ACCESS_KEY}
|
||||
|
||||
spring.ai.bedrock.cohere.chat.enabled=true
|
||||
spring.ai.bedrock.cohere.chat.options.temperature=0.8
|
||||
----
|
||||
|
||||
TIP: replace the `regions`, `access-key` and `secret-key` with your AWS credentials.
|
||||
|
||||
This will create a `BedrockCohereChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the chat client for text generations.
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@RestController
|
||||
public class ChatController {
|
||||
|
||||
private final BedrockCohereChatClient chatClient;
|
||||
|
||||
@Autowired
|
||||
public ChatController(BedrockCohereChatClient chatClient) {
|
||||
this.chatClient = chatClient;
|
||||
}
|
||||
|
||||
@GetMapping("/ai/generate")
|
||||
public Map generate(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
return Map.of("generation", chatClient.call(message));
|
||||
}
|
||||
|
||||
@GetMapping("/open-ai/generateStream")
|
||||
public Flux<ChatResponse> generateStream(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
Prompt prompt = new Prompt(new UserMessage(message));
|
||||
return chatClient.stream(prompt);
|
||||
}
|
||||
}
|
||||
----
|
||||
|
||||
== Manual Configuration
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/cohere/BedrockCohereChatClient.java[BedrockCohereChatClient] implements the `ChatClient` and `StreamingChatClient` and uses the <<low-level-api>> to connect to the Bedrock Anthropic service.
|
||||
|
||||
Add the `spring-ai-bedrock` dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-bedrock</artifactId>
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
----
|
||||
|
||||
or to your Gradle `build.gradle` build file.
|
||||
|
||||
[source,gradle]
|
||||
----
|
||||
dependencies {
|
||||
implementation 'org.springframework.ai:spring-ai-bedrock:0.8.0-SNAPSHOT'
|
||||
}
|
||||
----
|
||||
|
||||
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
|
||||
Next, create an https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/cohere/BedrockCohereChatClient.java[BedrockCohereChatClient] and use it for text generations:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
CohereChatBedrockApi api = new CohereChatBedrockApi(CohereChatModel.COHERE_COMMAND_V14.id(),
|
||||
EnvironmentVariableCredentialsProvider.create(), Region.US_EAST_1.id(), new ObjectMapper());
|
||||
|
||||
BedrockCohereChatClient chatClient = new BedrockCohereChatClient(api,
|
||||
BedrockCohereChatOptions.builder()
|
||||
.withTemperature(0.6f)
|
||||
.withTopK(10)
|
||||
.withTopP(0.5f)
|
||||
.withMaxTokens(678)
|
||||
.build()
|
||||
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt("Generate the names of 5 famous pirates."));
|
||||
|
||||
// Or with streaming responses
|
||||
Flux<ChatResponse> response = chatClient.stream(
|
||||
new Prompt("Generate the names of 5 famous pirates."));
|
||||
----
|
||||
|
||||
== Low-level CohereChatBedrockApi Client [[low-level-api]]
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/cohere/api/CohereChatBedrockApi.java[CohereChatBedrockApi] provides is lightweight Java client on top of AWS Bedrock https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command.html[Cohere Command models].
|
||||
|
||||
Following class diagram illustrates the CohereChatBedrockApi interface and building blocks:
|
||||
|
||||
image::bedrock/bedrock-cohere-chat-api.jpg[CohereChatBedrockApi Class Diagram]
|
||||
|
||||
The CohereChatBedrockApi supports the `cohere.command-light-text-v14` and `cohere.command-text-v14` models for both synchronous (e.g. `chatCompletion()`) and streaming (e.g. `chatCompletionStream()`) requests.
|
||||
|
||||
Here is a simple snippet how to use the api programmatically:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
CohereChatBedrockApi cohereChatApi = new CohereChatBedrockApi(
|
||||
CohereChatModel.COHERE_COMMAND_V14.id(),
|
||||
Region.US_EAST_1.id());
|
||||
|
||||
var request = CohereChatRequest
|
||||
.builder("What is the capital of Bulgaria and what is the size? What it the national anthem?")
|
||||
.withStream(false)
|
||||
.withTemperature(0.5f)
|
||||
.withTopP(0.8f)
|
||||
.withTopK(15)
|
||||
.withMaxTokens(100)
|
||||
.withStopSequences(List.of("END"))
|
||||
.withReturnLikelihoods(CohereChatRequest.ReturnLikelihoods.ALL)
|
||||
.withNumGenerations(3)
|
||||
.withLogitBias(null)
|
||||
.withTruncate(Truncate.NONE)
|
||||
.build();
|
||||
|
||||
CohereChatResponse response = cohereChatApi.chatCompletion(request);
|
||||
|
||||
var request = CohereChatRequest
|
||||
.builder("What is the capital of Bulgaria and what is the size? What it the national anthem?")
|
||||
.withStream(true)
|
||||
.withTemperature(0.5f)
|
||||
.withTopP(0.8f)
|
||||
.withTopK(15)
|
||||
.withMaxTokens(100)
|
||||
.withStopSequences(List.of("END"))
|
||||
.withReturnLikelihoods(CohereChatRequest.ReturnLikelihoods.ALL)
|
||||
.withNumGenerations(3)
|
||||
.withLogitBias(null)
|
||||
.withTruncate(Truncate.NONE)
|
||||
.build();
|
||||
|
||||
Flux<CohereChatResponse.Generation> responseStream = cohereChatApi.chatCompletionStream(request);
|
||||
List<CohereChatResponse.Generation> responses = responseStream.collectList().block();
|
||||
----
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ Refer to the xref:api/clients/bedrock.adoc[Spring AI documentation on Amazon Bed
|
||||
|
||||
== Auto-configuration
|
||||
|
||||
or you can leverage the `spring-ai-bedrock-ai-spring-boot-starter` Spring Boot starter:
|
||||
Add the `spring-ai-bedrock-ai-spring-boot-starter` dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
@@ -37,8 +37,9 @@ dependencies {
|
||||
|
||||
=== Enable Llama2 Chat Support
|
||||
|
||||
Spring AI defines a configuration property named `spring.ai.bedrock.llama2.chat.enabled` that you should set to `true` to enable support for Llama2.
|
||||
Exporting environment variables in one way to set this configuration property.
|
||||
By default the Bedrock Llama2 model is disabled.
|
||||
To enable it set the `spring.ai.bedrock.llama2.chat.enabled` property to `true`.
|
||||
Exporting environment variable is one way to set this configuration property:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
@@ -59,7 +60,7 @@ The prefix `spring.ai.bedrock.aws` is the property prefix to configure the conne
|
||||
|====
|
||||
|
||||
|
||||
The prefix `spring.ai.bedrock.llama2.chat` is the property prefix that configures the `ChatClient` implementation for Llama2.
|
||||
The prefix `spring.ai.bedrock.llama2.chat` is the property prefix that configures the chat client implementation for Llama2.
|
||||
|
||||
[cols="2,5,1"]
|
||||
|====
|
||||
@@ -72,45 +73,83 @@ The prefix `spring.ai.bedrock.llama2.chat` is the property prefix that configure
|
||||
| spring.ai.bedrock.llama2.chat.options.max-gen-len | Specify the maximum number of tokens to use in the generated response. The model truncates the response once the generated text exceeds maxGenLen. | 300
|
||||
|====
|
||||
|
||||
Look at the Spring AI enumeration, `Llama2ChatModel` for other model IDs. The other value supported is `meta.llama2-13b-chat-v1`.
|
||||
|
||||
Look at https://github.com/spring-projects/spring-ai/blob/4ba9a3cd689b9fd3a3805f540debe398a079c6ef/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/llama2/api/Llama2ChatBedrockApi.java#L164[Llama2ChatBedrockApi#Llama2ChatModel] for other model IDs. The other value supported is `meta.llama2-13b-chat-v1`.
|
||||
Model ID values can also be found in the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html[AWS Bedrock documentation for base model IDs].
|
||||
|
||||
=== Sample Code
|
||||
TIP: All properties prefixed with `spring.ai.bedrock.llama2.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.
|
||||
|
||||
This will create a `ChatClient` implementation that you can inject into your class.
|
||||
=== Chat Options [[chat-options]]
|
||||
|
||||
Create an `application.properties` file in the `src/main/resources` directory and add the following properties to configure the Llama2 Chat client.
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/llama2/BedrockLlama2ChatOptions.java[BedrockLlama2ChatOptions.java] provides model configurations, such as temperature, topK, topP, etc.
|
||||
|
||||
On start-up, the default options can be configured with the `BedrockLlama2ChatClient(api, options)` constructor or the `spring.ai.bedrock.llama2.chat.options.*` properties.
|
||||
|
||||
At run-time you can override the default options by adding new, request specific, options to the `Prompt` call.
|
||||
For example to override the default temperature for a specific request:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
BedrockLlama2ChatOptions.builder()
|
||||
.withTemperature(0.4)
|
||||
.build()
|
||||
));
|
||||
----
|
||||
|
||||
TIP: In addition to the model specific https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/llama2/BedrockLlama2ChatOptions.java[BedrockLlama2ChatOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptions.java[ChatOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptionsBuilder.java[ChatOptionsBuilder#builder()].
|
||||
|
||||
=== Sample Controller (Auto-configuration)
|
||||
|
||||
https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-bedrock-ai-spring-boot-starter` to your pom (or gradle) dependencies.
|
||||
|
||||
Add a `application.properties` file, under the `src/main/resources` directory, to enable and configure the Anthropic Chat client:
|
||||
|
||||
[source]
|
||||
----
|
||||
spring.ai.bedrock.aws.region=eu-central-1
|
||||
spring.ai.bedrock.aws.access-key=${AWS_ACCESS_KEY_ID}
|
||||
spring.ai.bedrock.aws.secret-key=${AWS_SECRET_ACCESS_KEY}
|
||||
|
||||
spring.ai.bedrock.llama2.chat.enabled=true
|
||||
spring.ai.bedrock.llama2.chat.options.temperature=0.8
|
||||
----
|
||||
|
||||
Here is an example of a simple `@Controller` class that uses the `ChatClient` implementation.
|
||||
TIP: replace the `regions`, `access-key` and `secret-key` with your AWS credentials.
|
||||
|
||||
This will create a `BedrockLlama2ChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the chat client for text generations.
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@RestController
|
||||
public class ChatController {
|
||||
|
||||
private final ChatClient chatClient;
|
||||
private final BedrockLlama2ChatClient chatClient;
|
||||
|
||||
@Autowired
|
||||
public ChatController(ChatClient chatClient) {
|
||||
public ChatController(BedrockLlama2ChatClient chatClient) {
|
||||
this.chatClient = chatClient;
|
||||
}
|
||||
|
||||
@GetMapping("/ai/generate")
|
||||
public Map generate(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
return Map.of("generation", chatClient.generate(message));
|
||||
return Map.of("generation", chatClient.call(message));
|
||||
}
|
||||
|
||||
@GetMapping("/open-ai/generateStream")
|
||||
public Flux<ChatResponse> generateStream(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
Prompt prompt = new Prompt(new UserMessage(message));
|
||||
return chatClient.stream(prompt);
|
||||
}
|
||||
}
|
||||
----
|
||||
|
||||
== Manual Configuration
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/llama2/BedrockLlama2ChatClient.java[BedrockLlama2ChatClient] implements the `ChatClient` and `StreamingChatClient` and uses the <<low-level-api>> to connect to the Bedrock Anthropic service.
|
||||
|
||||
Add the `spring-ai-bedrock` dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
[source,xml]
|
||||
@@ -131,11 +170,9 @@ dependencies {
|
||||
}
|
||||
----
|
||||
|
||||
NOTE: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
|
||||
The link:./src/main/java/org/springframework/ai/bedrock/llama2/BedrockLlama2ChatClient.java[BedrockLlama2ChatClient] implements the `ChatClient` and `StreamingChatClient` and uses the `Llama2ChatBedrockApi` library to connect to the Bedrock Llama2 service.
|
||||
|
||||
Here is how to create and use a `BedrockLlama2ChatClient`:
|
||||
Next, create an https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/llama2/BedrockLlama2ChatClient.java[BedrockLlama2ChatClient] and use it for text generations:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@@ -143,10 +180,10 @@ Llama2ChatBedrockApi api = new Llama2ChatBedrockApi(Llama2ChatModel.LLAMA2_70B_C
|
||||
EnvironmentVariableCredentialsProvider.create(), Region.US_EAST_1.id(), new ObjectMapper());
|
||||
|
||||
BedrockLlama2ChatClient chatClient = new BedrockLlama2ChatClient(api,
|
||||
BedrockLlama2ChatOptions.builder()
|
||||
.withTemperature(0.5f)
|
||||
.withMaxGenLen(100)
|
||||
.withTopP(0.9f).build());
|
||||
BedrockLlama2ChatOptions.builder()
|
||||
.withTemperature(0.5f)
|
||||
.withMaxGenLen(100)
|
||||
.withTopP(0.9f).build());
|
||||
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt("Generate the names of 5 famous pirates."));
|
||||
@@ -156,28 +193,23 @@ Flux<ChatResponse> response = chatClient.stream(
|
||||
new Prompt("Generate the names of 5 famous pirates."));
|
||||
----
|
||||
|
||||
== Low-level Llama2ChatBedrockApi Client [[low-level-api]]
|
||||
|
||||
== Appendices
|
||||
|
||||
=== Using low-level Llama2ChatBedrockApi Library
|
||||
|
||||
link:./src/main/java/org/springframework/ai/bedrock/llama2/api/Llama2ChatBedrockApi.java[Llama2ChatBedrockApi] provides is lightweight Java client on top of AWS Bedrock https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html[Meta Llama 2 and Llama 2 Chat models].
|
||||
https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/llama2/api/Llama2ChatBedrockApi.java[Llama2ChatBedrockApi] provides is lightweight Java client on top of AWS Bedrock https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html[Meta Llama 2 and Llama 2 Chat models].
|
||||
|
||||
Following class diagram illustrates the Llama2ChatBedrockApi interface and building blocks:
|
||||
|
||||
image::bedrock/bedrock-llama2-chat-api.jpg[Llama2ChatBedrockApi Class Diagram]
|
||||
|
||||
The Llama2ChatBedrockApi supports the `meta.llama2-13b-chat-v1` and `meta.llama2-70b-chat-v1` models.
|
||||
|
||||
Also the Llama2ChatBedrockApi supports both synchronous (e.g. `chatCompletion()`) and streaming (e.g. `chatCompletionStream()`) responses.
|
||||
The Llama2ChatBedrockApi supports the `meta.llama2-13b-chat-v1` and `meta.llama2-70b-chat-v1` models for both synchronous (e.g. `chatCompletion()`) and streaming (e.g. `chatCompletionStream()`) responses.
|
||||
|
||||
Here is a simple snippet how to use the api programmatically:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
Llama2ChatBedrockApi llama2ChatApi = new Llama2ChatBedrockApi(
|
||||
Llama2ChatModel.LLAMA2_70B_CHAT_V1.id(),
|
||||
Region.US_EAST_1.id());
|
||||
Llama2ChatModel.LLAMA2_70B_CHAT_V1.id(),
|
||||
Region.US_EAST_1.id());
|
||||
|
||||
Llama2ChatRequest request = Llama2ChatRequest.builder("Hello, my name is")
|
||||
.withTemperature(0.9f)
|
||||
@@ -187,16 +219,11 @@ Llama2ChatRequest request = Llama2ChatRequest.builder("Hello, my name is")
|
||||
|
||||
Llama2ChatResponse response = llama2ChatApi.chatCompletion(request);
|
||||
|
||||
System.out.println(response.generation());
|
||||
|
||||
// Streaming response
|
||||
Flux<Llama2ChatResponse> responseStream = llama2ChatApi.chatCompletionStream(request);
|
||||
|
||||
List<Llama2ChatResponse> responses = responseStream.collectList().block();
|
||||
|
||||
System.out.println(responses);
|
||||
----
|
||||
|
||||
Follow the link:./src/main/java/org/springframework/ai/bedrock/llama2/api/Llama2ChatBedrockApi.java[Llama2ChatBedrockApi.java]'s JavaDoc for further information.
|
||||
Follow the https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/llama2/api/Llama2ChatBedrockApi.java[Llama2ChatBedrockApi.java]'s JavaDoc for further information.
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
= HuggingFace
|
||||
= HuggingFace Chat
|
||||
|
||||
HuggingFace Inference Endpoints allow you to deploy and serve machine learning models in the cloud, making them accessible via an API.
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ Spring AI supports the Ollama text generation with `OllamaChatClient`.
|
||||
You first need to run Ollama on your local machine.
|
||||
Refer to the official Ollama project link:https://github.com/jmorganca/ollama[README] to get started running models on your local machine.
|
||||
|
||||
Note, installing `ollama run llama2` will download a 4GB docker image.
|
||||
NOTE: installing `ollama run llama2` will download a 4GB docker image.
|
||||
|
||||
== Auto-configuration
|
||||
|
||||
@@ -33,7 +33,7 @@ dependencies {
|
||||
}
|
||||
----
|
||||
|
||||
NOTE: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
|
||||
=== Chat Properties
|
||||
|
||||
@@ -46,9 +46,9 @@ The prefix `spring.ai.ollama` is the property prefix to configure the connection
|
||||
| spring.ai.ollama.base-url | Base URL where Ollama API server is running. | `http://localhost:11434`
|
||||
|====
|
||||
|
||||
The prefix `spring.ai.ollama.chat.options` is the property prefix that configures the `ChatClient` implementation for Ollama.
|
||||
The prefix `spring.ai.ollama.chat.options` is the property prefix that configures the chat client implementation for Ollama.
|
||||
|
||||
NOTE: The listed properties are based on the https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values[Ollama Valid Parameters and Values] and https://github.com/jmorganca/ollama/blob/main/api/types.go[Ollama Types]. And the default values are based on: https://github.com/ollama/ollama/blob/b538dc3858014f94b099730a592751a5454cab0a/api/types.go#L364[Ollama type defaults].
|
||||
NOTE: The `options` properties are based on the link:https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values[Ollama Valid Parameters and Values] and link:https://github.com/jmorganca/ollama/blob/main/api/types.go[Ollama Types]. The default values are based on: link:https://github.com/ollama/ollama/blob/b538dc3858014f94b099730a592751a5454cab0a/api/types.go#L364[Ollama type defaults].
|
||||
|
||||
[cols="3,6,1"]
|
||||
|====
|
||||
@@ -94,34 +94,80 @@ NOTE: The listed properties are based on the https://github.com/jmorganca/ollama
|
||||
|
||||
NOTE: The list of options for chat is to be reviewed. This https://github.com/spring-projects/spring-ai/issues/230[issue] will track progress.
|
||||
|
||||
=== Sample Code
|
||||
TIP: All properties prefixed with `spring.ai.ollama.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.
|
||||
|
||||
This will create a `ChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the `ChatClient` implementation.
|
||||
=== Chat Options [[chat-options]]
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java[OllamaOptions.java] provides model configurations, such as the model to use, the temperature, etc.
|
||||
|
||||
On start-up, the default options can be configured with the `OllamaChatClient(api, options)` constructor or the `spring.ai.ollama.chat.options.*` properties.
|
||||
|
||||
At run-time you can override the default options by adding new, request specific, options to the `Prompt` call.
|
||||
For example to override the default model and temperature for a specific request:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
OllamaOptions.create()
|
||||
.withModel("llama2")
|
||||
.withTemperature(0.4)
|
||||
));
|
||||
----
|
||||
|
||||
TIP: In addition to the model specific link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java[OllamaOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptions.java[ChatOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptionsBuilder.java[ChatOptionsBuilder#builder()].
|
||||
|
||||
=== Sample Controller (Auto-configuration)
|
||||
|
||||
https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-openai-spring-boot-starter` to your pom (or gradle) dependencies.
|
||||
|
||||
Add a `application.properties` file, under the `src/main/resources` directory, to enable and configure the OpenAi Chat client:
|
||||
|
||||
[source,application.properties]
|
||||
----
|
||||
spring.ai.ollama.base-url=http://localhost:11434
|
||||
spring.ai.ollama.chat.model=mistral
|
||||
spring.ai.ollama.chat.options.temperature=0.7
|
||||
----
|
||||
|
||||
TIP: replace the `base-url` with your Ollama server URL.
|
||||
|
||||
This will create a `OllamaChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the chat client for text generations.
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@RestController
|
||||
public class ChatController {
|
||||
|
||||
private final ChatClient chatClient;
|
||||
private final OllamaChatClient chatClient;
|
||||
|
||||
@Autowired
|
||||
public ChatController(ChatClient chatClient) {
|
||||
public ChatController(OllamaChatClient chatClient) {
|
||||
this.chatClient = chatClient;
|
||||
}
|
||||
|
||||
@GetMapping("/ai/generate")
|
||||
public Map generate(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
return Map.of("generation", chatClient.generate(message));
|
||||
return Map.of("generation", chatClient.call(message));
|
||||
}
|
||||
|
||||
@GetMapping("/open-ai/generateStream")
|
||||
public Flux<ChatResponse> generateStream(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
Prompt prompt = new Prompt(new UserMessage(message));
|
||||
return chatClient.stream(prompt);
|
||||
}
|
||||
|
||||
}
|
||||
----
|
||||
|
||||
== Manual Configuration
|
||||
|
||||
If you don't want to use the Spring Boot auto-configuration, you can manually configure the `OllamaChatClient` in your application.
|
||||
For this add the spring-ai-ollama dependency to your project’s Maven pom.xml file:
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java[OllamaChatClient] implements the `ChatClient` and `StreamingChatClient` and uses the <<low-level-api>> to connect to the Ollama service.
|
||||
|
||||
To use it add the `spring-ai-ollama` dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
@@ -141,7 +187,7 @@ dependencies {
|
||||
}
|
||||
----
|
||||
|
||||
NOTE: The `spring-ai-ollama` dependency provides access also to the `OllamaEmbeddingClient`.
|
||||
TIP: The `spring-ai-ollama` dependency provides access also to the `OllamaEmbeddingClient`.
|
||||
For more information about the `OllamaEmbeddingClient` refer to the link:../embeddings/ollama-embeddings.html[Ollama Embedding Client] section.
|
||||
|
||||
Next, create an `OllamaChatClient` instance and use it to text generations requests:
|
||||
@@ -165,27 +211,45 @@ Flux<ChatResponse> response = chatClient.stream(
|
||||
|
||||
The `OllamaOptions` provides the configuration information for all chat requests.
|
||||
|
||||
=== Chat Options
|
||||
=== Low-level OpenAiApi Client [[low-level-api]]
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java[OllamaOptions.java] provides provides configuration information for the chat requests, such as the model to use, the temperature, the frequency penalty, etc.
|
||||
The link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java[OllamaApi] provides is lightweight Java client for Ollama Chat API link:https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-chat-completion[Ollama Chat Completion API].
|
||||
|
||||
The default options can be configured using the `spring.ai.ollama.chat.options` properties as well.
|
||||
Following class diagram illustrates the `OllamaApi` chat interfaces and building blocks:
|
||||
|
||||
On start-time use the `OllamaChatClient#withDefaultOptions()` to set the default options applicable for all chat completion requests.
|
||||
At run-time you can override the default options with `OllamaOptions` instance in the request `Prompt`.
|
||||
image::ollama-chat-completion-api.png[OllamaApi Chat Completion API Diagram]
|
||||
|
||||
For example to override the default model name and temperature for a specific request:
|
||||
Here is a simple snippet how to use the api programmatically:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
OllamaOptions.create()
|
||||
.withModel("llama2")
|
||||
.withTemperature(0.4)
|
||||
));
|
||||
----
|
||||
OllamaApi ollamaApi =
|
||||
new OllamaApi("YOUR_HOST:YOUR_PORT");
|
||||
|
||||
You can use as prompt options any instance that implements the portable `ChatOptions` interface.
|
||||
For example you can use the `ChatOptionsBuilder` to create a portable prompt options.
|
||||
// Sync request
|
||||
var request = ChatRequest.builder("orca-mini")
|
||||
.withStream(false) // not streaming
|
||||
.withMessages(List.of(
|
||||
Message.builder(Role.SYSTEM)
|
||||
.withContent("You are geography teacher. You are talking to a student.")
|
||||
.build(),
|
||||
Message.builder(Role.USER)
|
||||
.withContent("What is the capital of Bulgaria and what is the size? "
|
||||
+ "What it the national anthem?")
|
||||
.build()))
|
||||
.withOptions(OllamaOptions.create().withTemperature(0.9f))
|
||||
.build();
|
||||
|
||||
ChatResponse response = ollamaApi.chat(request);
|
||||
|
||||
// Streaming request
|
||||
var request2 = ChatRequest.builder("orca-mini")
|
||||
.withStream(true) // streaming
|
||||
.withMessages(List.of(Message.builder(Role.USER)
|
||||
.withContent("What is the capital of Bulgaria and what is the size? " + "What it the national anthem?")
|
||||
.build()))
|
||||
.withOptions(OllamaOptions.create().withTemperature(0.9f).toMap())
|
||||
.build();
|
||||
|
||||
Flux<ChatResponse> streamingResponse = ollamaApi.streamingChat(request2);
|
||||
----
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Spring AI supports ChatGPT, the AI language model by OpenAI. ChatGPT has been instrumental in sparking interest in AI-driven text generation, thanks to its creation of industry-leading text generation models and embeddings.
|
||||
|
||||
== Pre-requisites
|
||||
== Prerequisites
|
||||
|
||||
You will need to create an API with OpenAI to access ChatGPT models.
|
||||
Create an account at https://platform.openai.com/signup[OpenAI signup page] and generate the token on the https://platform.openai.com/account/api-keys[API Keys page].
|
||||
@@ -37,7 +37,7 @@ dependencies {
|
||||
}
|
||||
----
|
||||
|
||||
NOTE: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
|
||||
=== Chat Properties
|
||||
|
||||
@@ -51,7 +51,7 @@ The prefix `spring.ai.openai` is used as the property prefix that lets you conne
|
||||
| spring.ai.openai.api-key | The API Key | -
|
||||
|====
|
||||
|
||||
The prefix `spring.ai.openai.chat` is the property prefix that lets you configure the `ChatClient` implementation for OpenAI.
|
||||
The prefix `spring.ai.openai.chat` is the property prefix that lets you configure the chat client implementation for OpenAI.
|
||||
|
||||
[cols="3,5,1"]
|
||||
|====
|
||||
@@ -59,7 +59,7 @@ The prefix `spring.ai.openai.chat` is the property prefix that lets you configur
|
||||
|
||||
| spring.ai.openai.chat.base-url | Optional overrides the spring.ai.openai.base-url to provide chat specific url | -
|
||||
| spring.ai.openai.chat.api-key | Optional overrides the spring.ai.openai.api-key to provide chat specific api-key | -
|
||||
| spring.ai.openai.chat.options.model | This is the OpenAI Chat model to use | `gpt-35-turbo` (the `gpt-3.5-turbo`, `gpt-4`, and `gpt-4-32k` point to the latest model versions)
|
||||
| spring.ai.openai.chat.options.model | This is the OpenAI Chat model to use | `gpt-3.5-turbo` (the `gpt-3.5-turbo`, `gpt-4`, and `gpt-4-32k` point to the latest model versions)
|
||||
| spring.ai.openai.chat.options.temperature | The sampling temperature to use that controls the apparent creativity of generated completions. Higher values will make output more random while lower values will make results more focused and deterministic. It is not recommended to modify temperature and top_p for the same completions request as the interaction of these two settings is difficult to predict. | 0.8
|
||||
| spring.ai.openai.chat.options.frequencyPenalty | Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. | 0.0f
|
||||
| spring.ai.openai.chat.options.logitBias | Modify the likelihood of specified tokens appearing in the completion. | -
|
||||
@@ -79,30 +79,59 @@ NOTE: You can override the common `spring.ai.openai.base-url` and `spring.ai.ope
|
||||
The `spring.ai.openai.chat.base-url` and `spring.ai.openai.chat.api-key` properties if set take precedence over the common properties.
|
||||
This is useful if you want to use different OpenAI accounts for different models and different model endpoints.
|
||||
|
||||
=== Sample Code
|
||||
TIP: All properties prefixed with `spring.ai.openai.chat.options` can be overridden at runtime by adding a request specific <<chat-options>> to the `Prompt` call.
|
||||
|
||||
This will create a `ChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the `ChatClient` implementation.
|
||||
=== Chat Options [[chat-options]]
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java[OpenAiChatOptions.java] provides model configurations, such as the model to use, the temperature, the frequency penalty, etc.
|
||||
|
||||
On start-up, the default options can be configured with the `OpenAiChatClient(api, options)` constructor or the `spring.ai.openai.chat.options.*` properties.
|
||||
|
||||
At run-time you can override the default options by adding new, request specific, options to the `Prompt` call.
|
||||
For example to override the default model and temperature for a specific request:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
OpenAiChatOptions.builder()
|
||||
.withModel("gpt-4-32k")
|
||||
.withTemperature(0.4)
|
||||
.build()
|
||||
));
|
||||
----
|
||||
|
||||
TIP: In addition to the model specific https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java[OpenAiChatOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptions.java[ChatOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/ChatOptionsBuilder.java[ChatOptionsBuilder#builder()].
|
||||
|
||||
=== Sample Controller (Auto-configuration)
|
||||
|
||||
https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-openai-spring-boot-starter` to your pom (or gradle) dependencies.
|
||||
|
||||
Add a `application.properties` file, under the `src/main/resources` directory, to enable and configure the OpenAi Chat client:
|
||||
|
||||
[source,application.properties]
|
||||
----
|
||||
spring.ai.openai.api-key=YOUR_API_KEY
|
||||
spring.ai.openai.chat.options.model=gpt-35-turbo
|
||||
spring.ai.openai.chat.options.model=gpt-3.5-turbo
|
||||
spring.ai.openai.chat.options.temperature=0.7
|
||||
----
|
||||
|
||||
TIP: replace the `api-key` with your OpenAI credentials.
|
||||
|
||||
This will create a `OpenAiChatClient` implementation that you can inject into your class.
|
||||
Here is an example of a simple `@Controller` class that uses the chat client for text generations.
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@RestController
|
||||
public class ChatController {
|
||||
|
||||
private final ChatClient chatClient;
|
||||
private final StreamingChatClient streamingChatClient;
|
||||
private final OpenAiChatClient chatClient;
|
||||
|
||||
@Autowired
|
||||
public ChatController(ChatClient chatClient, StreamingChatClient streamingChatClient) {
|
||||
public ChatController(OpenAiChatClient chatClient) {
|
||||
this.chatClient = chatClient;
|
||||
this.streamingChatClient = streamingChatClient;
|
||||
}
|
||||
|
||||
@GetMapping("/open-ai/generate")
|
||||
@@ -113,15 +142,17 @@ public class ChatController {
|
||||
@GetMapping("/open-ai/generateStream")
|
||||
public Flux<ChatResponse> generateStream(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
|
||||
Prompt prompt = new Prompt(new UserMessage(message));
|
||||
return streamingChatClient.stream(prompt);
|
||||
return chatClient.stream(prompt);
|
||||
}
|
||||
}
|
||||
----
|
||||
|
||||
== Manual Configuration
|
||||
|
||||
If you are not using Spring Boot, you can manually configure the `OpenAiChatClient` by creating the beans in your configuration class.
|
||||
For this add the `spring-ai-openai` dependency to your project's Maven `pom.xml` file:
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatClient.java[OpenAiChatClient] implements the `ChatClient` and `StreamingChatClient` and uses the <<low-level-api>> to connect to the OpenAI service.
|
||||
|
||||
Add the `spring-ai-openai` dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
[source, xml]
|
||||
----
|
||||
<dependency>
|
||||
@@ -140,9 +171,9 @@ dependencies {
|
||||
}
|
||||
----
|
||||
|
||||
NOTE: The `spring-ai-openai` dependency provides access also to the `OpenAiEmbeddingClient`. For more information about the `OpenAiEmbeddingClient` refer to the link:../embeddings/openai-embeddings.html[OpenAI Embeddings Client] section.
|
||||
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
|
||||
|
||||
Next, create an `OpenAiChatClient` instance and use it to text generations requests:
|
||||
Next, create a `OpenAiChatClient` and use it for text generations:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
@@ -166,28 +197,35 @@ Flux<ChatResponse> response = chatClient.stream(
|
||||
The `OpenAiChatOptions` provides the configuration information for the chat requests.
|
||||
The `OpenAiChatOptions.Builder` is fluent options builder.
|
||||
|
||||
=== Chat Options
|
||||
=== Low-level OpenAiApi Client [[low-level-api]]
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java[OpenAiChatOptions.java] provides provides the configuration information for the chat requests, such as the model to use, the temperature, the frequency penalty, etc.
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java[OpenAiApi] provides is lightweight Java client for OpenAI Chat API link:https://platform.openai.com/docs/api-reference/chat[OpenAI Chat API].
|
||||
|
||||
The default options can be configured using the `spring.ai.openai.chat.options` properties as well.
|
||||
On start-time use the `OpenAiChatClient#withDefaultOptions()` to set the default options applicable for all chat completion requests.
|
||||
At run-time you can override the default options with `OpenAiChatOptions` instance in the request `Prompt`.
|
||||
Following class diagram illustrates the `OpenAiApi` chat interfaces and building blocks:
|
||||
|
||||
For example to override the default model name and temperature for a specific request:
|
||||
image::openai-chat-api.png[OpenAiApi Chat API Diagram]
|
||||
|
||||
Here is a simple snippet how to use the api programmatically:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ChatResponse response = chatClient.call(
|
||||
new Prompt(
|
||||
"Generate the names of 5 famous pirates.",
|
||||
AzureOpenAiChatOptions.builder()
|
||||
.withModel("gpt-4-32k")
|
||||
.withTemperature(0.4)
|
||||
.build()
|
||||
));
|
||||
OpenAiApi openAiApi =
|
||||
new OpenAiApi(System.getenv("OPENAI_API_KEY"));
|
||||
|
||||
ChatCompletionMessage chatCompletionMessage =
|
||||
new ChatCompletionMessage("Hello world", Role.USER);
|
||||
|
||||
// Sync request
|
||||
ResponseEntity<ChatCompletion> response = openAiApi.chatCompletionEntity(
|
||||
new ChatCompletionRequest(List.of(chatCompletionMessage), "gpt-3.5-turbo", 0.8f, false));
|
||||
|
||||
// Streaming request
|
||||
Flux<ChatCompletionChunk> streamResponse = openAiApi.chatCompletionStream(
|
||||
new ChatCompletionRequest(List.of(chatCompletionMessage), "gpt-3.5-turbo", 0.8f, true));
|
||||
----
|
||||
|
||||
You can use as prompt options any instance that implements the portable `ChatOptions` interface.
|
||||
For example you can use the `ChatOptionsBuilder` to create a portable prompt options.
|
||||
Check the link:https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/api/OpenAiApiIT.java[OpenAiApiIT.java] integration test for more examples.
|
||||
|
||||
Follow the https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java[OpenAiApi.java]'s JavaDoc for further information.
|
||||
|
||||
|
||||
|
||||
@@ -51,8 +51,8 @@ public class AzureOpenAiAutoConfiguration {
|
||||
public AzureOpenAiChatClient azureOpenAiChatClient(OpenAIClient openAIClient,
|
||||
AzureOpenAiChatProperties chatProperties) {
|
||||
|
||||
AzureOpenAiChatClient azureOpenAiChatClient = new AzureOpenAiChatClient(openAIClient)
|
||||
.withDefaultOptions(chatProperties.getOptions());
|
||||
AzureOpenAiChatClient azureOpenAiChatClient = new AzureOpenAiChatClient(openAIClient,
|
||||
chatProperties.getOptions());
|
||||
|
||||
return azureOpenAiChatClient;
|
||||
}
|
||||
|
||||
@@ -59,16 +59,7 @@ public class BedrockCohereChatAutoConfiguration {
|
||||
public BedrockCohereChatClient cohereChatClient(CohereChatBedrockApi cohereChatApi,
|
||||
BedrockCohereChatProperties properties) {
|
||||
|
||||
LogitBias logitBiasBias = (properties.getLogitBiasBias() != null && properties.getLogitBiasToken() != null)
|
||||
? new LogitBias(properties.getLogitBiasToken(), properties.getLogitBiasBias()) : null;
|
||||
|
||||
return new BedrockCohereChatClient(cohereChatApi).withTemperature(properties.getTemperature())
|
||||
.withTopP(properties.getTopP())
|
||||
.withTopK(properties.getTopK())
|
||||
.withMaxTokens(properties.getMaxTokens())
|
||||
.withStopSequences(properties.getStopSequences())
|
||||
.withLogitBias(logitBiasBias)
|
||||
.withTruncate(properties.getTruncate());
|
||||
return new BedrockCohereChatClient(cohereChatApi, properties.getOptions());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -16,12 +16,10 @@
|
||||
|
||||
package org.springframework.ai.autoconfigure.bedrock.cohere;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.ai.bedrock.cohere.BedrockCohereChatOptions;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.ReturnLikelihoods;
|
||||
import org.springframework.ai.bedrock.cohere.api.CohereChatBedrockApi.CohereChatRequest.Truncate;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
|
||||
/**
|
||||
* Bedrock Cohere Chat autoconfiguration properties.
|
||||
@@ -44,67 +42,11 @@ public class BedrockCohereChatProperties {
|
||||
*/
|
||||
private String model = CohereChatBedrockApi.CohereChatModel.COHERE_COMMAND_V14.id();
|
||||
|
||||
/**
|
||||
* (optional) Use a lower value to decrease randomness in the response. Defaults to
|
||||
* 0.7.
|
||||
*/
|
||||
private Float temperature = 0.7f;
|
||||
|
||||
/**
|
||||
* (optional) The maximum cumulative probability of tokens to consider when sampling.
|
||||
* The generative uses combined Top-k and nucleus sampling. Nucleus sampling considers
|
||||
* the smallest set of tokens whose probability sum is at least topP.
|
||||
*/
|
||||
private Float topP;
|
||||
|
||||
/**
|
||||
* (optional) Specify the number of token choices the generative uses to generate the
|
||||
* next token.
|
||||
*/
|
||||
private Integer topK;
|
||||
|
||||
/**
|
||||
* (optional) Specify the maximum number of tokens to use in the generated response.
|
||||
*/
|
||||
private Integer maxTokens;
|
||||
|
||||
/**
|
||||
* (optional) Configure up to four sequences that the generative recognizes. After a
|
||||
* stop sequence, the generative stops generating further tokens. The returned text
|
||||
* doesn't contain the stop sequence.
|
||||
*/
|
||||
private List<String> stopSequences;
|
||||
|
||||
/**
|
||||
* (optional) Specify how and if the token likelihoods are returned with the response.
|
||||
*/
|
||||
private ReturnLikelihoods returnLikelihoods;
|
||||
|
||||
/**
|
||||
* (optional) The maximum number of generations that the generative should return.
|
||||
*/
|
||||
private Integer numGenerations;
|
||||
|
||||
/**
|
||||
* LogitBias prevents the generative from generating unwanted tokens or incentivize
|
||||
* the generative to include desired tokens. The token likelihoods.
|
||||
*/
|
||||
private String logitBiasToken;
|
||||
|
||||
/**
|
||||
* LogitBias prevents the generative from generating unwanted tokens or incentivize
|
||||
* the generative to include desired tokens. A float between -10 and 10.
|
||||
*/
|
||||
private Float logitBiasBias;
|
||||
|
||||
/**
|
||||
* (optional) Specifies how the API handles inputs longer than the maximum token
|
||||
* length.
|
||||
*/
|
||||
private Truncate truncate;
|
||||
@NestedConfigurationProperty
|
||||
private BedrockCohereChatOptions options = BedrockCohereChatOptions.builder().build();
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
return this.enabled;
|
||||
}
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
@@ -112,91 +54,19 @@ public class BedrockCohereChatProperties {
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
return this.model;
|
||||
}
|
||||
|
||||
public void setModel(String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
public Float getTemperature() {
|
||||
return temperature;
|
||||
public BedrockCohereChatOptions getOptions() {
|
||||
return this.options;
|
||||
}
|
||||
|
||||
public void setTemperature(Float temperature) {
|
||||
this.temperature = temperature;
|
||||
}
|
||||
|
||||
public Float getTopP() {
|
||||
return topP;
|
||||
}
|
||||
|
||||
public void setTopP(Float topP) {
|
||||
this.topP = topP;
|
||||
}
|
||||
|
||||
public Integer getTopK() {
|
||||
return topK;
|
||||
}
|
||||
|
||||
public void setTopK(Integer topK) {
|
||||
this.topK = topK;
|
||||
}
|
||||
|
||||
public Integer getMaxTokens() {
|
||||
return maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokens(Integer maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
public List<String> getStopSequences() {
|
||||
return stopSequences;
|
||||
}
|
||||
|
||||
public void setStopSequences(List<String> stopSequences) {
|
||||
this.stopSequences = stopSequences;
|
||||
}
|
||||
|
||||
public ReturnLikelihoods getReturnLikelihoods() {
|
||||
return returnLikelihoods;
|
||||
}
|
||||
|
||||
public void setReturnLikelihoods(ReturnLikelihoods returnLikelihoods) {
|
||||
this.returnLikelihoods = returnLikelihoods;
|
||||
}
|
||||
|
||||
public Integer getNumGenerations() {
|
||||
return numGenerations;
|
||||
}
|
||||
|
||||
public void setNumGenerations(Integer numGenerations) {
|
||||
this.numGenerations = numGenerations;
|
||||
}
|
||||
|
||||
public String getLogitBiasToken() {
|
||||
return logitBiasToken;
|
||||
}
|
||||
|
||||
public void setLogitBiasToken(String logitBiasToken) {
|
||||
this.logitBiasToken = logitBiasToken;
|
||||
}
|
||||
|
||||
public Float getLogitBiasBias() {
|
||||
return logitBiasBias;
|
||||
}
|
||||
|
||||
public void setLogitBiasBias(Float logitBiasBias) {
|
||||
this.logitBiasBias = logitBiasBias;
|
||||
}
|
||||
|
||||
public Truncate getTruncate() {
|
||||
return truncate;
|
||||
}
|
||||
|
||||
public void setTruncate(Truncate truncate) {
|
||||
this.truncate = truncate;
|
||||
public void setOptions(BedrockCohereChatOptions options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -60,8 +60,7 @@ public class OpenAiAutoConfiguration {
|
||||
|
||||
var openAiApi = new OpenAiApi(baseUrl, apiKey, restClientBuilder);
|
||||
|
||||
OpenAiChatClient openAiChatClient = new OpenAiChatClient(openAiApi)
|
||||
.withDefaultOptions(chatProperties.getOptions());
|
||||
OpenAiChatClient openAiChatClient = new OpenAiChatClient(openAiApi, chatProperties.getOptions());
|
||||
|
||||
return openAiChatClient;
|
||||
}
|
||||
|
||||
@@ -56,7 +56,8 @@ public class BedrockCohereChatAutoConfigurationIT {
|
||||
"spring.ai.bedrock.aws.secret-key=" + System.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
"spring.ai.bedrock.aws.region=" + Region.US_EAST_1.id(),
|
||||
"spring.ai.bedrock.cohere.chat.model=" + CohereChatModel.COHERE_COMMAND_V14.id(),
|
||||
"spring.ai.bedrock.cohere.chat.temperature=0.5", "spring.ai.bedrock.cohere.chat.maxTokens=500")
|
||||
"spring.ai.bedrock.cohere.chat.options.temperature=0.5",
|
||||
"spring.ai.bedrock.cohere.chat.options.maxTokens=500")
|
||||
.withConfiguration(AutoConfigurations.of(BedrockCohereChatAutoConfiguration.class));
|
||||
|
||||
private final Message systemMessage = new SystemPromptTemplate("""
|
||||
@@ -103,14 +104,18 @@ public class BedrockCohereChatAutoConfigurationIT {
|
||||
@Test
|
||||
public void propertiesTest() {
|
||||
|
||||
new ApplicationContextRunner().withPropertyValues("spring.ai.bedrock.cohere.chat.enabled=true",
|
||||
"spring.ai.bedrock.aws.access-key=ACCESS_KEY", "spring.ai.bedrock.aws.secret-key=SECRET_KEY",
|
||||
"spring.ai.bedrock.cohere.chat.model=MODEL_XYZ",
|
||||
"spring.ai.bedrock.aws.region=" + Region.EU_CENTRAL_1.id(),
|
||||
"spring.ai.bedrock.cohere.chat.temperature=0.55", "spring.ai.bedrock.cohere.chat.topP=0.55",
|
||||
"spring.ai.bedrock.cohere.chat.topK=10", "spring.ai.bedrock.cohere.chat.stopSequences=END1,END2",
|
||||
"spring.ai.bedrock.cohere.chat.returnLikelihoods=ALL", "spring.ai.bedrock.cohere.chat.numGenerations=3",
|
||||
"spring.ai.bedrock.cohere.chat.truncate=START", "spring.ai.bedrock.cohere.chat.maxTokens=123")
|
||||
new ApplicationContextRunner()
|
||||
.withPropertyValues("spring.ai.bedrock.cohere.chat.enabled=true",
|
||||
"spring.ai.bedrock.aws.access-key=ACCESS_KEY", "spring.ai.bedrock.aws.secret-key=SECRET_KEY",
|
||||
"spring.ai.bedrock.cohere.chat.model=MODEL_XYZ",
|
||||
"spring.ai.bedrock.aws.region=" + Region.EU_CENTRAL_1.id(),
|
||||
"spring.ai.bedrock.cohere.chat.options.temperature=0.55",
|
||||
"spring.ai.bedrock.cohere.chat.options.topP=0.55", "spring.ai.bedrock.cohere.chat.options.topK=10",
|
||||
"spring.ai.bedrock.cohere.chat.options.stopSequences=END1,END2",
|
||||
"spring.ai.bedrock.cohere.chat.options.returnLikelihoods=ALL",
|
||||
"spring.ai.bedrock.cohere.chat.options.numGenerations=3",
|
||||
"spring.ai.bedrock.cohere.chat.options.truncate=START",
|
||||
"spring.ai.bedrock.cohere.chat.options.maxTokens=123")
|
||||
.withConfiguration(AutoConfigurations.of(BedrockCohereChatAutoConfiguration.class))
|
||||
.run(context -> {
|
||||
var chatProperties = context.getBean(BedrockCohereChatProperties.class);
|
||||
@@ -120,14 +125,14 @@ public class BedrockCohereChatAutoConfigurationIT {
|
||||
assertThat(aswProperties.getRegion()).isEqualTo(Region.EU_CENTRAL_1.id());
|
||||
assertThat(chatProperties.getModel()).isEqualTo("MODEL_XYZ");
|
||||
|
||||
assertThat(chatProperties.getTemperature()).isEqualTo(0.55f);
|
||||
assertThat(chatProperties.getTopP()).isEqualTo(0.55f);
|
||||
assertThat(chatProperties.getTopK()).isEqualTo(10);
|
||||
assertThat(chatProperties.getStopSequences()).isEqualTo(List.of("END1", "END2"));
|
||||
assertThat(chatProperties.getReturnLikelihoods()).isEqualTo(ReturnLikelihoods.ALL);
|
||||
assertThat(chatProperties.getNumGenerations()).isEqualTo(3);
|
||||
assertThat(chatProperties.getTruncate()).isEqualTo(Truncate.START);
|
||||
assertThat(chatProperties.getMaxTokens()).isEqualTo(123);
|
||||
assertThat(chatProperties.getOptions().getTemperature()).isEqualTo(0.55f);
|
||||
assertThat(chatProperties.getOptions().getTopP()).isEqualTo(0.55f);
|
||||
assertThat(chatProperties.getOptions().getTopK()).isEqualTo(10);
|
||||
assertThat(chatProperties.getOptions().getStopSequences()).isEqualTo(List.of("END1", "END2"));
|
||||
assertThat(chatProperties.getOptions().getReturnLikelihoods()).isEqualTo(ReturnLikelihoods.ALL);
|
||||
assertThat(chatProperties.getOptions().getNumGenerations()).isEqualTo(3);
|
||||
assertThat(chatProperties.getOptions().getTruncate()).isEqualTo(Truncate.START);
|
||||
assertThat(chatProperties.getOptions().getMaxTokens()).isEqualTo(123);
|
||||
|
||||
assertThat(aswProperties.getAccessKey()).isEqualTo("ACCESS_KEY");
|
||||
assertThat(aswProperties.getSecretKey()).isEqualTo("SECRET_KEY");
|
||||
|
||||
Reference in New Issue
Block a user