Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve Ollama Options support

Browse Source 
- Rename OllamaChatClient#withOptions(...) method to OllamaChatClient#withDefaultOptions(...)
 - Rename OllamaEmbeddingClient#withOptions(...) method to OllamaEmbeddingClient#withDefaultOptions(...)
 - Remove the  Chat/Embedding Client model field by defaultOptions.mode one.
 - Correct default and runtime OllamaOptions merging implemented.
 - Added support for portable ChatOptions.
 - OllamaOptions a synthetic field ‘model’ not supported by Ollama API but used by the OllamaChatClient
   and OllamaEbeddingClients. The model field is removed before calling the OllamaApi.
 - Update the IT ollama docker image to 0.1.23
 - Set Mistral as the default model.
 - Extend and improve the ITs
 - Add tests for testing the chat and embedding request creation and options merging logic.
 - Minor code-style improvements.
 - Split the ollama.adoc into embeddings/ollama-embeddings.addoc and clients/ollama-chat.adoc.
 - Improve the documentation to explain how to configure and use the Ollama Chat and Embedding clients manually or with the help of the auto-configuraitons.
 - Clarify the docs property sections.
This commit is contained in:
Christian Tzolov
2024-02-03 14:57:54 +01:00
parent 60a60b4610
commit 97537e0a10
22 changed files with 702 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2023 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,33 +16,28 @@
package org.springframework.ai.ollama;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import reactor.core.publisher.Flux;
import org.springframework.ai.chat.ChatClient;
import org.springframework.ai.chat.ChatOptions;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
import org.springframework.ai.chat.StreamingChatClient;
import org.springframework.ai.chat.metadata.Usage;
import org.springframework.ai.ollama.api.OllamaApi;
import org.springframework.ai.ollama.api.OllamaOptions;
import org.springframework.ai.ollama.api.OllamaApi.ChatRequest;
import org.springframework.ai.ollama.api.OllamaApi.Message.Role;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.MessageType;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.metadata.Usage;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.model.ModelOptionsUtils;
import org.springframework.ai.ollama.api.OllamaApi;
import org.springframework.ai.ollama.api.OllamaApi.Message.Role;
import org.springframework.ai.ollama.api.OllamaOptions;
import org.springframework.util.StringUtils;
/**
* {@link ChatClient} implementation for {@literal Ollma}.
* {@link ChatClient} implementation for {@literal Ollama}.
*
* Ollama allows developers to run large language models and generate embeddings locally.
* It supports open-source models available on [Ollama AI
@@ -57,37 +52,38 @@ import org.springframework.ai.chat.messages.MessageType;
*/
public class OllamaChatClient implements ChatClient, StreamingChatClient {
/**
* Low-level Ollama API library.
*/
private final OllamaApi chatApi;
private String model = "orca-mini";
private Map<String, Object> clientOptions;
private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
/**
* Default options to be used for all chat requests.
*/
private OllamaOptions defaultOptions = OllamaOptions.create().withModel(OllamaOptions.DEFAULT_MODEL);
public OllamaChatClient(OllamaApi chatApi) {
this.chatApi = chatApi;
}
/**
* @deprecated Use {@link OllamaOptions#setModel} instead.
*/
@Deprecated
public OllamaChatClient withModel(String model) {
this.model = model;
this.defaultOptions.setModel(model);
return this;
}
public OllamaChatClient withOptions(Map<String, Object> options) {
this.clientOptions = options;
return this;
}
public OllamaChatClient withOptions(OllamaOptions options) {
this.clientOptions = options.toMap();
public OllamaChatClient withDefaultOptions(OllamaOptions options) {
this.defaultOptions = options;
return this;
}
@Override
public ChatResponse call(Prompt prompt) {
OllamaApi.ChatResponse response = this.chatApi.chat(request(prompt, this.model, false));
OllamaApi.ChatResponse response = this.chatApi.chat(ollamaChatRequest(prompt, false));
var generator = new Generation(response.message().content());
if (response.promptEvalCount() != null && response.evalCount() != null) {
generator = generator
@@ -99,7 +95,7 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient {
@Override
public Flux<ChatResponse> stream(Prompt prompt) {
Flux<OllamaApi.ChatResponse> response = this.chatApi.streamingChat(request(prompt, this.model, true));
Flux<OllamaApi.ChatResponse> response = this.chatApi.streamingChat(ollamaChatRequest(prompt, true));
return response.map(chunk -> {
Generation generation = (chunk.message() != null) ? new Generation(chunk.message().content())
@@ -127,7 +123,10 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient {
};
}
private OllamaApi.ChatRequest request(Prompt prompt, String model, boolean stream) {
/**
* Package access for testing.
*/
OllamaApi.ChatRequest ollamaChatRequest(Prompt prompt, boolean stream) {
List<OllamaApi.Message> ollamaMessages = prompt.getInstructions()
.stream()
@@ -138,51 +137,33 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient {
.toList();
// runtime options
Map<String, Object> clientOptionsToUse = merge(prompt.getOptions(), this.clientOptions, HashMap.class);
OllamaOptions runtimeOptions = null;
if (prompt.getOptions() != null) {
if (prompt.getOptions() instanceof ChatOptions runtimeChatOptions) {
runtimeOptions = ModelOptionsUtils.copyToTarget(runtimeChatOptions, ChatOptions.class,
OllamaOptions.class);
}
else {
throw new IllegalArgumentException("Prompt options are not of type ChatOptions: "
+ prompt.getOptions().getClass().getSimpleName());
}
}
return ChatRequest.builder(model)
OllamaOptions mergedOptions = ModelOptionsUtils.merge(runtimeOptions, this.defaultOptions, OllamaOptions.class);
// Override the model.
if (!StringUtils.hasText(mergedOptions.getModel())) {
throw new IllegalArgumentException("Model is not set!");
}
String model = mergedOptions.getModel();
return OllamaApi.ChatRequest.builder(model)
.withStream(stream)
.withMessages(ollamaMessages)
.withOptions(clientOptionsToUse)
.withOptions(mergedOptions)
.build();
}
public static Map<String, Object> objectToMap(Object source) {
try {
String json = OBJECT_MAPPER.writeValueAsString(source);
return OBJECT_MAPPER.readValue(json, new TypeReference<Map<String, Object>>() {
});
}
catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
public static <T> T mapToClass(Map<String, Object> source, Class<T> clazz) {
try {
String json = OBJECT_MAPPER.writeValueAsString(source);
return OBJECT_MAPPER.readValue(json, clazz);
}
catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
public static <T> T merge(Object source, Object target, Class<T> clazz) {
if (source == null) {
source = Map.of();
}
Map<String, Object> sourceMap = objectToMap(source);
Map<String, Object> targetMap = objectToMap(target);
targetMap.putAll(sourceMap.entrySet()
.stream()
.filter(e -> e.getValue() != null)
.collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())));
return mapToClass(targetMap, clazz);
}
private OllamaApi.Message.Role toRole(Message message) {
switch (message.getMessageType()) {

74
models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingClient.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2023 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,23 +18,26 @@ package org.springframework.ai.ollama;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.embedding.AbstractEmbeddingClient;
import org.springframework.ai.embedding.Embedding;
import org.springframework.ai.embedding.EmbeddingClient;
import org.springframework.ai.embedding.EmbeddingOptions;
import org.springframework.ai.embedding.EmbeddingResponse;
import org.springframework.ai.model.ModelOptionsUtils;
import org.springframework.ai.ollama.api.OllamaApi;
import org.springframework.ai.ollama.api.OllamaApi.EmbeddingRequest;
import org.springframework.ai.ollama.api.OllamaOptions;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
/**
* {@link EmbeddingClient} implementation for {@literal Ollma}.
* {@link EmbeddingClient} implementation for {@literal Ollama}.
*
* Ollama allows developers to run large language models and generate embeddings locally.
* It supports open-source models available on [Ollama AI
@@ -43,12 +46,11 @@ import org.springframework.util.Assert;
* Examples of models supported: - Llama 2 (7B parameters, 3.8GB size) - Mistral (7B
* parameters, 4.1GB size)
*
*
*
* Please refer to the <a href="https://ollama.ai/">official Ollama website</a> for the
* most up-to-date information on available models.
*
* @author Christian Tzolov
* @since 0.8.0
*/
public class OllamaEmbeddingClient extends AbstractEmbeddingClient {
@@ -56,26 +58,26 @@ public class OllamaEmbeddingClient extends AbstractEmbeddingClient {
private final OllamaApi ollamaApi;
private String model = "orca-mini";
private Map<String, Object> clientOptions;
/**
* Default options to be used for all chat requests.
*/
private OllamaOptions defaultOptions = OllamaOptions.create().withModel(OllamaOptions.DEFAULT_MODEL);
public OllamaEmbeddingClient(OllamaApi ollamaApi) {
this.ollamaApi = ollamaApi;
}
/**
* @deprecated Use {@link OllamaOptions#setModel} instead.
*/
@Deprecated
public OllamaEmbeddingClient withModel(String model) {
this.model = model;
this.defaultOptions.setModel(model);
return this;
}
public OllamaEmbeddingClient withOptions(Map<String, Object> options) {
this.clientOptions = options;
return this;
}
public OllamaEmbeddingClient withOptions(OllamaOptions options) {
this.clientOptions = options.toMap();
public OllamaEmbeddingClient withDefaultOptions(OllamaOptions options) {
this.defaultOptions = options;
return this;
}
@@ -94,15 +96,51 @@ public class OllamaEmbeddingClient extends AbstractEmbeddingClient {
List<List<Double>> embeddingList = new ArrayList<>();
for (String inputContent : request.getInstructions()) {
OllamaApi.EmbeddingResponse response = this.ollamaApi
.embeddings(new EmbeddingRequest(this.model, inputContent, this.clientOptions));
var ollamaEmbeddingRequest = ollamaEmbeddingRequest(inputContent, request.getOptions());
OllamaApi.EmbeddingResponse response = this.ollamaApi.embeddings(ollamaEmbeddingRequest);
embeddingList.add(response.embedding());
}
var indexCounter = new AtomicInteger(0);
List<Embedding> embeddings = embeddingList.stream()
.map(e -> new Embedding(e, indexCounter.getAndIncrement()))
.toList();
return new EmbeddingResponse(embeddings);
}
/**
* Package access for testing.
*/
OllamaApi.EmbeddingRequest ollamaEmbeddingRequest(String inputContent, EmbeddingOptions options) {
// runtime options
OllamaOptions runtimeOptions = null;
if (options != null) {
if (options instanceof OllamaOptions ollamaOptions) {
runtimeOptions = ollamaOptions;
}
else if (options instanceof EmbeddingOptions embeddingOptions) {
// currently EmbeddingOptions does not have any portable options to be
// merged.
runtimeOptions = null;
}
else {
throw new IllegalArgumentException("Request embedding options are not of type EmbeddingOptions: "
+ options.getClass().getSimpleName());
}
}
OllamaOptions mergedOptions = ModelOptionsUtils.merge(runtimeOptions, this.defaultOptions, OllamaOptions.class);
// Override the model.
if (!StringUtils.hasText(mergedOptions.getModel())) {
throw new IllegalArgumentException("Model is not set!");
}
String model = mergedOptions.getModel();
return new EmbeddingRequest(model, inputContent, OllamaOptions.filterNonSupportedFields(mergedOptions.toMap()));
}
}

11
models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2023 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import com.fasterxml.jackson.annotation.JsonInclude;
@@ -445,19 +446,21 @@ public class OllamaApi {
}
public Builder withOptions(Map<String, Object> options) {
this.options = options;
Objects.requireNonNullElse(options, "The options can not be null.");
this.options = OllamaOptions.filterNonSupportedFields(options);
return this;
}
public Builder withOptions(OllamaOptions options) {
this.options = options.toMap();
Objects.requireNonNullElse(options, "The options can not be null.");
this.options = OllamaOptions.filterNonSupportedFields(options.toMap());
return this;
}
public ChatRequest build() {
return new ChatRequest(model, messages, stream, format, options);
}
}
}

39
models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2023 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package org.springframework.ai.ollama.api;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
@@ -25,7 +26,9 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.ai.chat.ChatOptions;
import org.springframework.ai.embedding.EmbeddingOptions;
/**
* Helper class for creating strongly-typed Ollama options.
@@ -39,7 +42,9 @@ import org.springframework.ai.chat.ChatOptions;
* Types</a>
*/
@JsonInclude(Include.NON_NULL)
public class OllamaOptions implements ChatOptions {
public class OllamaOptions implements ChatOptions, EmbeddingOptions {
public static final String DEFAULT_MODEL = "mistral";
// @formatter:off
/**
@@ -233,6 +238,24 @@ public class OllamaOptions implements ChatOptions {
@JsonProperty("stop") private List<String> stop;
/**
* NOTE: Synthetic field not part of the official Ollama API.
* Used to allow overriding the model name with prompt options.
*/
@JsonProperty("model") private String model;
public OllamaOptions withModel(String model) {
this.model = model;
return this;
}
public String getModel() {
return model;
}
public void setModel(String model) {
this.model = model;
}
public OllamaOptions withUseNUMA(Boolean useNUMA) {
this.useNUMA = useNUMA;
@@ -686,6 +709,18 @@ public class OllamaOptions implements ChatOptions {
return new OllamaOptions();
}
/**
* Filter out the non supported fields from the options.
* @param options The options to filter.
* @return The filtered options.
*/
public static Map<String, Object> filterNonSupportedFields(Map<String, Object> options) {
return options.entrySet().stream()
.filter(e -> !e.getKey().equals("model"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
// @formatter:on
}

31
models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java
View File

@@ -1,3 +1,18 @@
/*
* Copyright 2024-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.ollama;
import java.io.IOException;
@@ -47,7 +62,7 @@ class OllamaChatClientIT {
private static final Log logger = LogFactory.getLog(OllamaChatClientIT.class);
@Container
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.21").withExposedPorts(11434);
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
static String baseUrl;
@@ -75,15 +90,19 @@ class OllamaChatClientIT {
UserMessage userMessage = new UserMessage("Tell me about 5 famous pirates from the Golden Age of Piracy.");
// portable/generic options
var chatOptionsBuilder = ChatOptionsBuilder.builder();
var portableOptions = ChatOptionsBuilder.builder().withTemperature(0.7f).build();
Prompt prompt = new Prompt(List.of(userMessage, systemMessage), portableOptions);
ChatResponse response = client.call(prompt);
assertThat(response.getResult().getOutput().getContent()).contains("Blackbeard");
// ollama specific options
var ollamaOptions = new OllamaOptions().withLowVRAM(true);
Prompt prompt = new Prompt(List.of(userMessage, systemMessage),
chatOptionsBuilder.withTemperature(0.7f).build());
ChatResponse response = client.call(prompt);
response = client.call(new Prompt(List.of(userMessage, systemMessage), ollamaOptions));
assertThat(response.getResult().getOutput().getContent()).contains("Blackbeard");
}
@Test
@@ -189,7 +208,7 @@ class OllamaChatClientIT {
@Bean
public OllamaChatClient ollamaChat(OllamaApi ollamaApi) {
return new OllamaChatClient(ollamaApi).withModel(MODEL)
.withOptions(OllamaOptions.create().withTemperature(0.9f));
.withDefaultOptions(OllamaOptions.create().withModel(MODEL).withTemperature(0.9f));
}
}

124
models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatRequestTests.java Normal file
View File

@@ -0,0 +1,124 @@
/*
* Copyright 2024-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.ollama;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.ChatOptions;
import org.springframework.ai.chat.ChatOptionsBuilder;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.ollama.api.OllamaApi;
import org.springframework.ai.ollama.api.OllamaOptions;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
public class OllamaChatRequestTests {
OllamaChatClient client = new OllamaChatClient(new OllamaApi()).withDefaultOptions(
new OllamaOptions().withModel("MODEL_NAME").withTopK(99).withTemperature(66.6f).withNumGPU(1));
@Test
public void createRequestWithDefaultOptions() {
var request = client.ollamaChatRequest(new Prompt("Test message content"), false);
assertThat(request.messages()).hasSize(1);
assertThat(request.stream()).isFalse();
assertThat(request.model()).isEqualTo("MODEL_NAME");
assertThat(request.options().get("temperature")).isEqualTo(66.6);
assertThat(request.options().get("top_k")).isEqualTo(99);
assertThat(request.options().get("num_gpu")).isEqualTo(1);
assertThat(request.options().get("top_p")).isNull();
}
@Test
public void createRequestWithPromptOllamaOptions() {
// Runtime options should override the default options.
OllamaOptions promptOptions = new OllamaOptions().withTemperature(0.8f).withTopP(0.5f).withNumGPU(2);
var request = client.ollamaChatRequest(new Prompt("Test message content", promptOptions), true);
assertThat(request.messages()).hasSize(1);
assertThat(request.stream()).isTrue();
assertThat(request.model()).isEqualTo("MODEL_NAME");
assertThat(request.options().get("temperature")).isEqualTo(0.8);
assertThat(request.options().get("top_k")).isEqualTo(99); // still the default
// value.
assertThat(request.options().get("num_gpu")).isEqualTo(2);
assertThat(request.options().get("top_p")).isEqualTo(0.5); // new field introduced
// by the
// promptOptions.
}
@Test
public void createRequestWithPromptPortableChatOptions() {
// Ollama runtime options.
ChatOptions portablePromptOptions = ChatOptionsBuilder.builder()
.withTemperature(0.9f)
.withTopK(100)
.withTopP(0.6f)
.build();
var request = client.ollamaChatRequest(new Prompt("Test message content", portablePromptOptions), true);
assertThat(request.messages()).hasSize(1);
assertThat(request.stream()).isTrue();
assertThat(request.model()).isEqualTo("MODEL_NAME");
assertThat(request.options().get("temperature")).isEqualTo(0.9);
assertThat(request.options().get("top_k")).isEqualTo(100);
assertThat(request.options().get("num_gpu")).isEqualTo(1); // default value.
assertThat(request.options().get("top_p")).isEqualTo(0.6);
}
@Test
public void createRequestWithPromptOptionsModelOverride() {
// Ollama runtime options.
OllamaOptions promptOptions = new OllamaOptions().withModel("PROMPT_MODEL");
var request = client.ollamaChatRequest(new Prompt("Test message content", promptOptions), true);
assertThat(request.model()).isEqualTo("PROMPT_MODEL");
}
@Test
public void createRequestWithDefaultOptionsModelOverride() {
OllamaChatClient client2 = new OllamaChatClient(new OllamaApi())
.withDefaultOptions(new OllamaOptions().withModel("DEFAULT_OPTIONS_MODEL"));
var request = client2.ollamaChatRequest(new Prompt("Test message content"), true);
assertThat(request.model()).isEqualTo("DEFAULT_OPTIONS_MODEL");
// Prompt options should override the default options.
OllamaOptions promptOptions = new OllamaOptions().withModel("PROMPT_MODEL");
request = client2.ollamaChatRequest(new Prompt("Test message content", promptOptions), true);
assertThat(request.model()).isEqualTo("PROMPT_MODEL");
}
}

17
models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java
View File

@@ -1,3 +1,18 @@
/*
* Copyright 2024-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.ollama;
import java.io.IOException;
@@ -30,7 +45,7 @@ class OllamaEmbeddingClientIT {
private static final Log logger = LogFactory.getLog(OllamaApiIT.class);
@Container
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.21").withExposedPorts(11434);
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
static String baseUrl;

64
models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingRequestTests.java Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright 2024-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.ollama;
import org.junit.jupiter.api.Test;
import org.springframework.ai.embedding.EmbeddingOptions;
import org.springframework.ai.ollama.api.OllamaApi;
import org.springframework.ai.ollama.api.OllamaOptions;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
public class OllamaEmbeddingRequestTests {
OllamaEmbeddingClient client = new OllamaEmbeddingClient(new OllamaApi()).withDefaultOptions(
new OllamaOptions().withModel("DEFAULT_MODEL").withMainGPU(11).withUseMMap(true).withNumGPU(1));
@Test
public void ollamaEmbeddingRequestDefaultOptions() {
var request = client.ollamaEmbeddingRequest("Hello", null);
assertThat(request.model()).isEqualTo("DEFAULT_MODEL");
assertThat(request.options().get("num_gpu")).isEqualTo(1);
assertThat(request.options().get("main_gpu")).isEqualTo(11);
assertThat(request.options().get("use_mmap")).isEqualTo(true);
assertThat(request.prompt()).isEqualTo("Hello");
}
@Test
public void ollamaEmbeddingRequestRequestOptions() {
EmbeddingOptions promptOptions = new OllamaOptions().withModel("PROMPT_MODEL")
.withMainGPU(22)
.withUseMMap(true)
.withNumGPU(2);
var request = client.ollamaEmbeddingRequest("Hello", promptOptions);
assertThat(request.model()).isEqualTo("PROMPT_MODEL");
assertThat(request.options().get("num_gpu")).isEqualTo(2);
assertThat(request.options().get("main_gpu")).isEqualTo(22);
assertThat(request.options().get("use_mmap")).isEqualTo(true);
assertThat(request.prompt()).isEqualTo("Hello");
}
}

2
models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java
View File

@@ -51,7 +51,7 @@ public class OllamaApiIT {
private static final Log logger = LogFactory.getLog(OllamaApiIT.class);
@Container
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.21").withExposedPorts(11434);
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
static OllamaApi ollamaApi;

2
models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatClient.java
View File

@@ -173,7 +173,7 @@ public class OpenAiChatClient implements ChatClient, StreamingChatClient {
request = ModelOptionsUtils.merge(updatedRuntimeOptions, request, ChatCompletionRequest.class);
}
else {
throw new IllegalArgumentException("Prompt options are not of type ChatCompletionRequest:"
throw new IllegalArgumentException("Prompt options are not of type ChatOptions: "
+ prompt.getOptions().getClass().getSimpleName());
}
}

9
spring-ai-core/src/main/java/org/springframework/ai/chat/prompt/Prompt.java
View File

@@ -64,8 +64,9 @@ public class Prompt implements ModelRequest<List<Message>> {
return sb.toString();
}
@Override
public ModelOptions getOptions() {
return modelOptions;
return this.modelOptions;
}
@Override
@@ -75,7 +76,7 @@ public class Prompt implements ModelRequest<List<Message>> {
@Override
public String toString() {
return "Prompt{" + "messages=" + messages + ", modelOptions=" + modelOptions + '}';
return "Prompt{" + "messages=" + this.messages + ", modelOptions=" + this.modelOptions + '}';
}
@Override
@@ -84,12 +85,12 @@ public class Prompt implements ModelRequest<List<Message>> {
return true;
if (!(o instanceof Prompt prompt))
return false;
return Objects.equals(messages, prompt.messages) && Objects.equals(modelOptions, prompt.modelOptions);
return Objects.equals(this.messages, prompt.messages) && Objects.equals(this.modelOptions, prompt.modelOptions);
}
@Override
public int hashCode() {
return Objects.hash(messages, modelOptions);
return Objects.hash(this.messages, this.modelOptions);
}
}

4
spring-ai-core/src/main/java/org/springframework/ai/model/ModelOptionsUtils.java
View File

@@ -72,6 +72,10 @@ public final class ModelOptionsUtils {
*/
public static <T> T merge(Object source, Object target, Class<T> clazz, List<String> acceptedFieldNames) {
if (source == null) {
source = Map.of();
}
List<String> requestFieldNames = CollectionUtils.isEmpty(acceptedFieldNames)
? REQUEST_FIELD_NAMES_PER_CLASS.computeIfAbsent(clazz, ModelOptionsUtils::getJsonPropertyValues)
: acceptedFieldNames;

3
spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc
View File

@@ -7,13 +7,14 @@
*** xref:api/clients/azure-openai.adoc[]
*** xref:api/clients/bedrock.adoc[]
*** xref:api/clients/huggingface.adoc[]
*** xref:api/clients/ollama.adoc[]
*** xref:api/clients/ollama-chat.adoc[]
** xref:api/prompt.adoc[]
** xref:api/output-parser.adoc[]
** xref:api/etl-pipeline.adoc[]
** xref:api/embeddings.adoc[]
*** xref:api/embeddings/onnx.adoc[]
*** xref:api/embeddings/openai-embeddings.adoc[]
*** xref:api/embeddings/ollama-embeddings.adoc[]
** xref:api/vectordbs.adoc[]
*** xref:api/vectordbs/azure.adoc[]
*** xref:api/vectordbs/chroma.adoc[]

146
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/ollama.adoc → spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/ollama-chat.adoc
View File

@@ -1,6 +1,7 @@
= Ollama
= Ollama Chat
Ollama lets you get up and running with large language models locally.
With https://ollama.ai/[Ollama] you can run various Large Language Models (LLMs) locally and generate text from them.
Spring AI supports the Ollama text generation with `OllamaChatClient`.
== Getting Started
@@ -10,11 +11,81 @@ Refer to the official Ollama project link:https://github.com/jmorganca/ollama[RE
Note, installing `ollama run llama2` will download a 4GB docker image.
== Project Dependencies
=== Configure the Ollama Chat Client Manually
Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
Add the spring-ai-ollama dependency to your project’s Maven pom.xml file:
Then add the Spring Boot Starter dependency to your project's Maven `pom.xml` build file:
[source,xml]
----
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-ollama</artifactId>
<version>0.8.0-SNAPSHOT</version>
</dependency>
----
or to your Gradle `build.gradle` build file.
[source,groovy]
----
dependencies {
implementation 'org.springframework.ai:spring-ai-ollama:0.8.0-SNAPSHOT'
}
----
NOTE: The `spring-ai-ollama` dependency provides access also to the `OllamaEmbeddingClient`.
For more information about the `OllamaEmbeddingClient` refer to the link:../embeddings/ollama-embeddings.html[Ollama Embedding Client] section.
Next, create an `OllamaChatClient` instance and use it to text generations requests:
[source,java]
----
var ollamaApi = new OllamaApi();
var chatClient = new OllamaChatClient(ollamaApi).withModel(MODEL)
.withDefaultOptions(OllamaOptions.create()
.withModel(OllamaOptions.DEFAULT_MODEL)
.withTemperature(0.9f));
ChatResponse response = chatClient.call(
new Prompt("Generate the names of 5 famous pirates."));
// Or with streaming responses
Flux<ChatResponse> response = chatClient.stream(
new Prompt("Generate the names of 5 famous pirates."));
----
The `OllamaOptions` provides the configuration information for all chat requests.
==== ChatOptions and OllamaOptions
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java[OllamaOptions.java] provides provides configuration information for the chat requests, such as the model to use, the temperature, the frequency penalty, etc.
The default options can be configured using the `spring.ai.ollama.chat.options` properties as well.
On start-time use the `OllamaChatClient#withDefaultOptions()` to set the default options applicable for all chat completion requests.
At run-time you can override the default options with `OllamaOptions` instance in the request `Prompt`.
For example to override the default model name and temperature for a specific request:
[source,java]
----
ChatResponse response = chatClient.call(
new Prompt(
"Generate the names of 5 famous pirates.",
OllamaOptions.create()
.withModel("llama2")
.withTemperature(0.4)
));
----
You can use as prompt options any instance that implements the portable `ChatOptions` interface.
For example you can use the `ChatOptionsBuilder` to create a portable prompt options.
=== OllamaChatClient Auto-configuration
Spring AI provides Spring Boot auto-configuration for the Ollama Chat Client.
To enable it add the following dependency to your project's Maven `pom.xml` file:
[source,xml]
----
@@ -34,7 +105,10 @@ dependencies {
}
----
== Sample Code
NOTE: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
==== Sample Code
This will create a `ChatClient` implementation that you can inject into your class.
Here is an example of a simple `@Controller` class that uses the `ChatClient` implementation.
@@ -62,26 +136,26 @@ public class ChatController {
The prefix `spring.ai.ollama` is the property prefix to configure the connection to Ollama
[cols="3,5,3"]
[cols="3,6,1"]
|====
| Property | Description | Default
| spring.ai.ollama.base-url | Base URL where Ollama API server is running. | `http://localhost:11434`
| spring.ai.ollama.model | Language model to use. | `llama2`
|====
[NOTE]
====
The list of options for chat and embedding is to be reviewd. This https://github.com/spring-projects/spring-ai/issues/230[issue] will track progress.
====
NOTE: The list of options for chat is to be reviewed. This https://github.com/spring-projects/spring-ai/issues/230[issue] will track progress.
The prefix `spring.ai.ollama.chat` is the property prefix that configures the `ChatClient` implementation for Ollama.
NOTE: The `spring.ai.ollama.chat.options.*` properties are based on the https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values[Ollama Valid Parameters and Values] and https://github.com/jmorganca/ollama/blob/main/api/types.go[Ollama Types]
[cols="3,5,3"]
The prefix `spring.ai.ollama.chat.options` is the property prefix that configures the `ChatClient` implementation for Ollama.
[cols="3,6,1"]
|====
| Property | Description | Default
| spring.ai.ollama.chat.model | The name of the model to use | llama2
| spring.ai.ollama.chat.model (DEPRECATED) | The name of the model to use. Deprecated use the `spring.ai.ollama.chat.options.model` instead | mistral
| spring.ai.ollama.chat.options.model | The name of the https://github.com/ollama/ollama?tab=readme-ov-file#model-library[supported models] to use. | mistral
| spring.ai.ollama.chat.options.numa | Whether to use NUMA. | false
| spring.ai.ollama.chat.options.num-ctx | Sets the size of the context window used to generate the next token. | 2048
| spring.ai.ollama.chat.options.num-batch | ??? | -
@@ -118,45 +192,3 @@ The prefix `spring.ai.ollama.chat` is the property prefix that configures the `C
|====
The prefix `spring.ai.ollama.embedding` is the property prefix that configures the `EmbeddingClient` implementation for Ollama.
[cols="3,5,3"]
|====
| Property | Description | Default
| spring.ai.ollama.embedding.model | The name of the model to use | llama2
| spring.ai.ollama.embedding.options.numa | Whether to use NUMA. | false
| spring.ai.ollama.embedding.options.num-ctx | Sets the size of the context window used to generate the next token. | 2048
| spring.ai.ollama.embedding.options.num-batch | ??? | -
| spring.ai.ollama.embedding.options.num-gqa | The number of GQA groups in the transformer layer. Required for some models, for example, it is 8 for llama2:70b. | -
| spring.ai.ollama.embedding.options.num-gpu | The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable. | -
| spring.ai.ollama.embedding.options.main-gpu | ??? | -
| spring.ai.ollama.embedding.options.low-vram | ??? | -
| spring.ai.ollama.embedding.options.f16-kv | ??? | -
| spring.ai.ollama.embedding.options.logits-all | ??? | -
| spring.ai.ollama.embedding.options.vocab-only | ??? | -
| spring.ai.ollama.embedding.options.use-mmap | ??? | -
| spring.ai.ollama.embedding.options.use-mlock | ??? | -
| spring.ai.ollama.embedding.options.embedding-only | ??? | -
| spring.ai.ollama.embedding.options.rope-frequency-base | ??? | -
| spring.ai.ollama.embedding.options.rope-frequency-scale | ??? | -
| spring.ai.ollama.chat.options.num-thread | Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). | -
| spring.ai.ollama.embedding.options.num-keep | ??? | -
| spring.ai.ollama.embedding.options.seed | Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. | 0
| spring.ai.ollama.embedding.options.num-predict | Maximum number of tokens to predict when generating text. (Default: 128, -1 = infinite generation, -2 = fill context) | 128
| spring.ai.ollama.embedding.options.top-k | Reduces the probability of generating nonsense. A higher value (e.g., 100) will give more diverse answers, while a lower value (e.g., 10) will be more conservative. | 40
| spring.ai.ollama.embedding.options.top-p | Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. | 0.9
| spring.ai.ollama.embedding.options.tfs-z | Tail-free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. | 1
| spring.ai.ollama.embedding.options.typical-p | ??? | -
| spring.ai.ollama.embedding.options.repeat-last-n | Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = num_ctx) | 64
| spring.ai.ollama.embedding.options.temperature | The temperature of the model. Increasing the temperature will make the model answer more creatively. | 0.8
| spring.ai.ollama.embedding.options.repeat-penalty | Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. | 1.1
| spring.ai.ollama.embedding.options.presence-penalty | ??? | -
| spring.ai.ollama.embedding.options.frequency-penalty | ??? | -
| spring.ai.ollama.embedding.options.mirostat | Enable Mirostat sampling for controlling perplexity. (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0) | 0
| spring.ai.ollama.embedding.options.mirostat-tau | Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. | 0.1
| spring.ai.ollama.embedding.options.mirostat-eta | Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. | 5.0
| spring.ai.ollama.embedding.options.penalize-newline | ??? | -
| spring.ai.ollama.embedding.options.stop | Sets the stop sequences to use. When this pattern is encountered the LLM will stop generating text and return. Multiple stop patterns may be set by specifying multiple separate stop parameters in a modelfile. | -
|====

2
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/openai-chat.adoc
View File

@@ -40,7 +40,7 @@ dependencies {
NOTE: The `spring-ai-openai` dependency provides access also to the `OpenAiEmbeddingClient`. For more information about the `OpenAiEmbeddingClient` refer to the link:../embeddings/openai-embeddings.html[OpenAI Embeddings Client] section.
Next, create an `OpenAiChatClient` instance and use it to compute the similarity between two input texts:
Next, create an `OpenAiChatClient` instance and use it to text generations requests:
[source,java]
----

187
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/embeddings/ollama-embeddings.adoc Normal file
View File

@@ -0,0 +1,187 @@
= Ollama Embeddings
With https://ollama.ai/[Ollama] you can run various Large Language Models (LLMs) locally and generate embeddings from them.
Spring AI supports the Ollama text embeddings with `OllamaEmbeddingClient`.
An embedding is a vector (list) of floating point numbers.
The distance between two vectors measures their relatedness.
Small distances suggest high relatedness and large distances suggest low relatedness.
== Getting Started
You first need to run Ollama on your local machine.
Refer to the official Ollama project link:https://github.com/jmorganca/ollama[README] to get started running models on your local machine.
Note, installing `ollama run llama2` will download a 4GB docker image.
=== Configure the Ollama Embedding Client Manually
Add the spring-ai-ollama dependency to your project’s Maven pom.xml file:
[source,xml]
----
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-ollama</artifactId>
<version>0.8.0-SNAPSHOT</version>
</dependency>
----
or to your Gradle `build.gradle` build file.
[source,groovy]
----
dependencies {
implementation 'org.springframework.ai:spring-ai-ollama:0.8.0-SNAPSHOT'
}
----
NOTE: The `spring-ai-ollama` dependency provides access also to the `OllamaChatClient`.
For more information about the `OllamaChatClient` refer to the link:../clients/ollama-chat.html[Ollama Chat Client] section.
Next, create an `OllamaEmbeddingClient` instance and use it to compute the similarity between two input texts:
[source,java]
----
var ollamaApi = new OllamaApi();
var embeddingClient = new OllamaEmbeddingClient(ollamaApi)
.withDefaultOptions(OllamaOptions.create()
.withModel(OllamaOptions.DEFAULT_MODEL)
.toMap());
EmbeddingResponse embeddingResponse = embeddingClient
.embedForResponse(List.of("Hello World", "World is big and salvation is near"));
----
The `OllamaOptions` provides the configuration information for all embedding requests.
==== OllamaOptions
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java[OllamaOptions.java] provides the Ollama configurations, such as the model to use, the low level GPU and CPU tunning, etc.
The default options can be configured using the `spring.ai.ollama.embedding.options` properties as well.
At start-time use the `OllamaEmbeddingClient#withDefaultOptions()` to configure the default options used for all embedding requests.
At run-time you can override the default options, using a `OllamaOptions` instance as part of your `EmbeddingRequest`.
For example to override the default model name for a specific request:
[source,java]
----
EmbeddingResponse embeddingResponse = embeddingClient.call(
new EmbeddingRequest(List.of("Hello World", "World is big and salvation is near"),
OllamaOptions.create()
.withModel("Different-Embedding-Model-Deployment-Name"));
----
=== OllamaEmbeddingClient Auto-configuration
Spring AI provides Spring Boot auto-configuration for the Azure Ollama Embedding Client.
To enable it add the following dependency to your Maven `pom.xml` file:
[source,xml]
----
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-ollama-spring-boot-starter</artifactId>
<version>0.8.0-SNAPSHOT</version>
</dependency>
----
or to your Gradle `build.gradle` build file.
[source,groovy]
----
dependencies {
implementation 'org.springframework.ai:spring-ai-ollama-spring-boot-starter:0.8.0-SNAPSHOT'
}
----
NOTE: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
The `spring.ai.ollama.embedding.options.*` properties are used to configure the default options used for all embedding requests.
(It is used as `OllamaEmbeddingClient#withDefaultOptions()` instance).
==== Sample Embedding Controller
This will create a `EmbeddingClient` implementation that you can inject into your class.
Here is an example of a simple `@Controller` class that uses the `EmbeddingClient` implementation.
[source,java]
----
@RestController
public class EmbeddingController {
private final EmbeddingClient embeddingClient;
@Autowired
public EmbeddingController(EmbeddingClient embeddingClient) {
this.embeddingClient = embeddingClient;
}
@GetMapping("/ai/embedding")
public Map embed(@RequestParam(value = "message", defaultValue = "Tell me a joke") String message) {
EmbeddingResponse embeddingResponse = this.embeddingClient.embedForResponse(List.of(message));
return Map.of("embedding", embeddingResponse);
}
}
----
== Ollama Embedding Properties
The prefix `spring.ai.ollama` is the property prefix to configure the connection to Ollama
[cols="3,6,2"]
|====
| Property | Description | Default
| spring.ai.ollama.base-url | Base URL where Ollama API server is running. | `http://localhost:11434`
|====
The prefix `spring.ai.ollama.embedding.options` is the property prefix that configures the `EmbeddingClient` implementation for Ollama.
[cols="3,6,1"]
|====
| Property | Description | Default
| spring.ai.ollama.embedding.model (DEPRECATED) | The name of the model to use. Deprecated use the `spring.ai.ollama.embedding.options.model` instead | mistral
| spring.ai.ollama.embedding.options.model | The name of the https://github.com/ollama/ollama?tab=readme-ov-file#model-library[supported models] to use. | mistral
| spring.ai.ollama.embedding.options.numa | Whether to use NUMA. | false
| spring.ai.ollama.embedding.options.num-ctx | Sets the size of the context window used to generate the next token. | 2048
| spring.ai.ollama.embedding.options.num-batch | ??? | -
| spring.ai.ollama.embedding.options.num-gqa | The number of GQA groups in the transformer layer. Required for some models, for example, it is 8 for llama2:70b. | -
| spring.ai.ollama.embedding.options.num-gpu | The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable. | -
| spring.ai.ollama.embedding.options.main-gpu | ??? | -
| spring.ai.ollama.embedding.options.low-vram | ??? | -
| spring.ai.ollama.embedding.options.f16-kv | ??? | -
| spring.ai.ollama.embedding.options.logits-all | ??? | -
| spring.ai.ollama.embedding.options.vocab-only | ??? | -
| spring.ai.ollama.embedding.options.use-mmap | ??? | -
| spring.ai.ollama.embedding.options.use-mlock | ??? | -
| spring.ai.ollama.embedding.options.embedding-only | ??? | -
| spring.ai.ollama.embedding.options.rope-frequency-base | ??? | -
| spring.ai.ollama.embedding.options.rope-frequency-scale | ??? | -
| spring.ai.ollama.chat.options.num-thread | Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). | -
| spring.ai.ollama.embedding.options.num-keep | ??? | -
| spring.ai.ollama.embedding.options.seed | Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. | 0
| spring.ai.ollama.embedding.options.num-predict | Maximum number of tokens to predict when generating text. (Default: 128, -1 = infinite generation, -2 = fill context) | 128
| spring.ai.ollama.embedding.options.top-k | Reduces the probability of generating nonsense. A higher value (e.g., 100) will give more diverse answers, while a lower value (e.g., 10) will be more conservative. | 40
| spring.ai.ollama.embedding.options.top-p | Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. | 0.9
| spring.ai.ollama.embedding.options.tfs-z | Tail-free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. | 1
| spring.ai.ollama.embedding.options.typical-p | ??? | -
| spring.ai.ollama.embedding.options.repeat-last-n | Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = num_ctx) | 64
| spring.ai.ollama.embedding.options.temperature | The temperature of the model. Increasing the temperature will make the model answer more creatively. | 0.8
| spring.ai.ollama.embedding.options.repeat-penalty | Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. | 1.1
| spring.ai.ollama.embedding.options.presence-penalty | ??? | -
| spring.ai.ollama.embedding.options.frequency-penalty | ??? | -
| spring.ai.ollama.embedding.options.mirostat | Enable Mirostat sampling for controlling perplexity. (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0) | 0
| spring.ai.ollama.embedding.options.mirostat-tau | Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. | 0.1
| spring.ai.ollama.embedding.options.mirostat-eta | Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. | 5.0
| spring.ai.ollama.embedding.options.penalize-newline | ??? | -
| spring.ai.ollama.embedding.options.stop | Sets the stop sequences to use. When this pattern is encountered the LLM will stop generating text and return. Multiple stop patterns may be set by specifying multiple separate stop parameters in a modelfile. | -
|====
NOTE: The `spring.ai.ollama.embedding.options.*` properties are based on the https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values[Ollama Valid Parameters and Values] and https://github.com/jmorganca/ollama/blob/main/api/types.go[Ollama Types]

9
spring-ai-docs/src/main/antora/modules/ROOT/pages/api/embeddings/openai-embeddings.adoc
View File

@@ -40,7 +40,8 @@ dependencies {
}
----
NOTE: The `spring-ai-openai` dependency provides access also to the `OpenAiChatClient`. For more information about the `OpenAiChatClient` refer to the link:../clients/openai-chat.html[OpenAI Chat Client] section.
NOTE: The `spring-ai-openai` dependency provides access also to the `OpenAiChatClient`.
For more information about the `OpenAiChatClient` refer to the link:../clients/openai-chat.html[OpenAI Chat Client] section.
Next, create an `OpenAiEmbeddingClient` instance and use it to compute the similarity between two input texts:
@@ -49,7 +50,7 @@ Next, create an `OpenAiEmbeddingClient` instance and use it to compute the simil
var openAiApi = new OpenAiApi(System.getenv("OPENAI_API_KEY"));
var embeddingClient = new OpenAiEmbeddingClient(openAiApi)
.withDefaultOptions(OpenAiEmbeddingOptions.builder()
.withDefaultOptions(OllamaOptions.create()
.withModel("text-embedding-ada-002")
.withUser("user-6")
.build());
@@ -63,12 +64,12 @@ The options class offers a `builder()` for easy options creation.
==== OpenAiEmbeddingOptions
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiEmbeddingOptions.java[OpenAiEmbeddingOptions.java] provides the OpenAI configures, such as the model to use, the temperature, the frequency penalty, etc.
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiEmbeddingOptions.java[OpenAiEmbeddingOptions.java] provides the OpenAI configurations, such as the model to use and etc.
The default options can be configured using the `spring.ai.openai.embedding.options` properties as well.
At start-time use the `OpenAiEmbeddingClient#withDefaultOptions()` to configure the default options used for all embedding requests.
At run-time you can override the default options, using a `OpenAiEmbeddingOptions` instance as part of your `EmbeddingRequest``.
At run-time you can override the default options, using a `OpenAiEmbeddingOptions` instance as part of your `EmbeddingRequest`.
For example to override the default model name for a specific request:

7
spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaAutoConfiguration.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023 the original author or authors.
* Copyright 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -48,7 +48,8 @@ public class OllamaAutoConfiguration {
@Bean
public OllamaChatClient ollamaChatClient(OllamaApi ollamaApi, OllamaChatProperties properties) {
return new OllamaChatClient(ollamaApi).withModel(properties.getModel()).withOptions(properties.getOptions());
return new OllamaChatClient(ollamaApi).withModel(properties.getModel())
.withDefaultOptions(properties.getOptions());
}
@Bean
@@ -56,7 +57,7 @@ public class OllamaAutoConfiguration {
public OllamaEmbeddingClient ollamaEmbeddingClient(OllamaApi ollamaApi, OllamaEmbeddingProperties properties) {
return new OllamaEmbeddingClient(ollamaApi).withModel(properties.getModel())
.withOptions(properties.getOptions());
.withDefaultOptions(properties.getOptions());
}
}

13
spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatProperties.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023 the original author or authors.
* Copyright 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -31,25 +31,20 @@ public class OllamaChatProperties {
public static final String CONFIG_PREFIX = "spring.ai.ollama.chat";
/**
* Ollama Chat generative name. Defaults to 'llama2'.
*/
private String model = "llama2";
/**
* Client lever Ollama options. Use this property to configure generative temperature,
* topK and topP and alike parameters. The null values are ignored defaulting to the
* generative's defaults.
*/
@NestedConfigurationProperty
private OllamaOptions options = new OllamaOptions();
private OllamaOptions options = OllamaOptions.create().withModel(OllamaOptions.DEFAULT_MODEL);
public String getModel() {
return this.model;
return this.options.getModel();
}
public void setModel(String model) {
this.model = model;
this.options.setModel(model);
}
public OllamaOptions getOptions() {

13
spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingProperties.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2023 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -31,25 +31,20 @@ public class OllamaEmbeddingProperties {
public static final String CONFIG_PREFIX = "spring.ai.ollama.embedding";
/**
* Ollama Embedding generative name. Defaults to 'llama2'.
*/
private String model = "llama2";
/**
* Client lever Ollama options. Use this property to configure generative temperature,
* topK and topP and alike parameters. The null values are ignored defaulting to the
* generative's defaults.
*/
@NestedConfigurationProperty
private OllamaOptions options = new OllamaOptions();
private OllamaOptions options = OllamaOptions.create().withModel(OllamaOptions.DEFAULT_MODEL);
public String getModel() {
return model;
return this.options.getModel();
}
public void setModel(String model) {
this.model = model;
this.options.setModel(model);
}
public OllamaOptions getOptions() {

18
spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaAutoConfigurationIT.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023-2023 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -53,10 +53,10 @@ public class OllamaAutoConfigurationIT {
private static final Log logger = LogFactory.getLog(OllamaAutoConfigurationIT.class);
private static String MODEL_NAME = "orca-mini";
private static String MODEL_NAME = "mistral";
@Container
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.16").withExposedPorts(11434);
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
static String baseUrl;
@@ -69,10 +69,14 @@ public class OllamaAutoConfigurationIT {
baseUrl = "http://" + ollamaContainer.getHost() + ":" + ollamaContainer.getMappedPort(11434);
}
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withPropertyValues("spring.ai.ollama.chat.enabled=true", "spring.ai.ollama.chat.model=" + MODEL_NAME,
"spring.ai.ollama.baseUrl=" + baseUrl, "spring.ai.ollama.chat.temperature=0.5",
"spring.ai.ollama.chat.topK=10")
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner().withPropertyValues(
// @formatter:off
"spring.ai.ollama.chat.enabled=true",
"spring.ai.ollama.chat.options.model=" + MODEL_NAME,
"spring.ai.ollama.baseUrl=" + baseUrl,
"spring.ai.ollama.chat.options.temperature=0.5",
"spring.ai.ollama.chat.options.topK=10")
// @formatter:on
.withConfiguration(AutoConfigurations.of(OllamaAutoConfiguration.class));
private final Message systemMessage = new SystemPromptTemplate("""

5
spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java
View File

@@ -47,7 +47,7 @@ public class OllamaEmbeddingAutoConfigurationIT {
private static String MODEL_NAME = "orca-mini";
@Container
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.16").withExposedPorts(11434);
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
static String baseUrl;
@@ -61,7 +61,8 @@ public class OllamaEmbeddingAutoConfigurationIT {
}
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withPropertyValues("spring.ai.ollama.embedding.model=" + MODEL_NAME, "spring.ai.ollama.base-url=" + baseUrl)
.withPropertyValues("spring.ai.ollama.embedding.options.model=" + MODEL_NAME,
"spring.ai.ollama.base-url=" + baseUrl)
.withConfiguration(AutoConfigurations.of(OllamaAutoConfiguration.class));
@Test