diff --git a/models/spring-ai-ollama/README.md b/models/spring-ai-ollama/README.md index d87417ed6..9738b0d02 100644 --- a/models/spring-ai-ollama/README.md +++ b/models/spring-ai-ollama/README.md @@ -22,7 +22,8 @@ EmbeddingResponse embeddings(EmbeddingRequest embeddingRequest) > NOTE: OllamaApi expose also the Ollama `generation` endpoint but later if inferior compared to the Ollama `chat` endpoint. -The `OllamaApiOptions` is helper class used as type-safe option builder. It provides `toMap` to convert the content into `Map`. +The `OllamaOptions` is helper class used as type-safe option builder. +It provides `toMap` to convert the content into `Map`. Here is a simple snippet how to use the OllamaApi programmatically: @@ -32,7 +33,7 @@ var request = ChatRequest.builder("orca-mini") .withMessages(List.of(Message.builder(Role.user) .withContent("What is the capital of Bulgaria and what is the size? " + "What it the national anthem?") .build())) - .withOptions(Options.builder().withTemperature(0.9f).build()) + .withOptions(Options.create().withTemperature(0.9f).withTopK(10)) .build(); ChatResponse response = ollamaApi.chat(request); @@ -44,7 +45,7 @@ var request = ChatRequest.builder("orca-mini") .withMessages(List.of(Message.builder(Role.user) .withContent("What is the capital of Bulgaria and what is the size? " + "What it the national anthem?") .build())) - .withOptions(Options.builder().withTemperature(0.9f).build().toMap()) + .withOptions(Options.create().withTemperature(0.9f)) .build(); Flux response = ollamaApi.streamingChat(request); @@ -76,15 +77,18 @@ public OllamaApi ollamaApi() { @Bean public OllamaChatClient ollamaChat(OllamaApi ollamaApi) { - return new OllamaChatClient(ollamaApi).withModel(MODEL) - .withOptions(OllamaApiOptions.Options.builder().withTemperature(0.9f).build()); + return new OllamaChatClient(ollamaApi) + .withModel("llama2") + .withOptions(OllamaOptions.create() + .withTemperature(0.9f) + .withTopK(12)); } @Bean public OllamaEmbeddingClient ollamaEmbedding(OllamaApi ollamaApi) { - return new OllamaEmbeddingClient(ollamaApi).withModel("orca-mini"); + return new OllamaEmbeddingClient(ollamaApi) + .withModel("orca-mini"); } - ``` or you can leverage the `spring-ai-ollama-spring-boot-starter` Spring Boot starter. @@ -98,23 +102,27 @@ For this add the following dependency: ``` +Use the `OllamaConnectionProperties` to configure the Ollama clients (both Chat and Embedding) connections: + +| Property | Description | Default | +| ------------- | ------------- | ------------- | +| spring.ai.ollama.base-url | The base url of the Ollama server. | http://localhost:11434 | + Use the `OllamaChatProperties` to configure the Ollama Chat client: | Property | Description | Default | | ------------- | ------------- | ------------- | | spring.ai.ollama.chat.model | Model to use. | llama2 | -| spring.ai.ollama.chat.base-url | The base url of the Ollama server. | http://localhost:11434 | | spring.ai.ollama.chat.enabled | Allows you to disable the Ollama Chat autoconfiguration. | true | | spring.ai.ollama.chat.temperature | Controls the randomness of the output. Values can range over [0.0,1.0] | 0.8 | | spring.ai.ollama.chat.topP | The maximum cumulative probability of tokens to consider when sampling. | - | | spring.ai.ollama.chat.topK | Max number or responses to generate. | - | -| spring.ai.options.chat.options (WIP) | A Map used to configure the Chat client. | - | +| spring.ai.options.chat.options | A `OllamaOptions` used to configure the Chat client. | - | and `OllamaEmbeddingProperties` to configure the Ollama Embedding client: | Property | Description | Default | | ------------- | ------------- | ------------- | | spring.ai.ollama.embedding.model | Model to use. | llama2 | -| spring.ai.ollama.embedding.base-url | The base url of the Ollama server. | http://localhost:11434 | | spring.ai.ollama.embedding.enabled | Allows you to disable the Ollama embedding autoconfiguration. | true | -| spring.ai.options.embedding.options (WIP) | A Map used to configure the embedding client. | - | +| spring.ai.options.embedding.options | `OllamaOptions` used to configure the embedding client. | - | diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java index d7283d8c5..122bfbd41 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatClient.java @@ -28,9 +28,10 @@ import org.springframework.ai.chat.StreamingChatClient; import org.springframework.ai.metadata.ChoiceMetadata; import org.springframework.ai.metadata.Usage; import org.springframework.ai.ollama.api.OllamaApi; +import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.ai.ollama.api.OllamaApi.ChatRequest; import org.springframework.ai.ollama.api.OllamaApi.Message.Role; -import org.springframework.ai.ollama.api.OllamaApiOptions; + import org.springframework.ai.prompt.Prompt; import org.springframework.ai.prompt.messages.Message; import org.springframework.ai.prompt.messages.MessageType; @@ -71,7 +72,7 @@ public class OllamaChatClient implements ChatClient, StreamingChatClient { return this; } - public OllamaChatClient withOptions(OllamaApiOptions.Options options) { + public OllamaChatClient withOptions(OllamaOptions options) { this.clientOptions = options.toMap(); return this; } diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingClient.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingClient.java index a37c9e834..32328e142 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingClient.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingClient.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import org.springframework.ai.chat.ChatClient; import org.springframework.ai.document.Document; import org.springframework.ai.embedding.AbstractEmbeddingClient; import org.springframework.ai.embedding.Embedding; @@ -28,7 +27,7 @@ import org.springframework.ai.embedding.EmbeddingClient; import org.springframework.ai.embedding.EmbeddingResponse; import org.springframework.ai.ollama.api.OllamaApi; import org.springframework.ai.ollama.api.OllamaApi.EmbeddingRequest; -import org.springframework.ai.ollama.api.OllamaApiOptions; +import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.util.Assert; /** @@ -70,7 +69,7 @@ public class OllamaEmbeddingClient extends AbstractEmbeddingClient { return this; } - public OllamaEmbeddingClient withOptions(OllamaApiOptions.Options options) { + public OllamaEmbeddingClient withOptions(OllamaOptions options) { this.clientOptions = options.toMap(); return this; } diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java index b17609494..c320afa74 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApi.java @@ -31,7 +31,6 @@ import org.apache.commons.logging.LogFactory; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import org.springframework.ai.ollama.api.OllamaApiOptions.Options; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.client.ClientHttpResponse; @@ -213,7 +212,7 @@ public class OllamaApi { return this; } - public Builder withOptions(Options options) { + public Builder withOptions(OllamaOptions options) { this.options = options.toMap(); return this; } @@ -450,7 +449,7 @@ public class OllamaApi { return this; } - public Builder withOptions(Options options) { + public Builder withOptions(OllamaOptions options) { this.options = options.toMap(); return this; } diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApiOptions.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApiOptions.java deleted file mode 100644 index 2f0beeba4..000000000 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaApiOptions.java +++ /dev/null @@ -1,395 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.ai.ollama.api; - -import java.util.Map; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -/** - * Helper class for building strongly typed the Ollama request options. - * - * @author Christian Tzolov - * @since 0.8.0 - */ -// @formatter:on -public class OllamaApiOptions { - - /** - * Runner options which must be set when the model is loaded into memory. - * - * @param useNUMA Whether to use NUMA. - * @param numCtx Sets the size of the context window used to generate the next token. - * (Default: 2048) - * @param numBatch ??? - * @param numGQA The number of GQA groups in the transformer layer. Required for some - * models, for example it is 8 for llama2:70b. - * @param numGPU The number of layers to send to the GPU(s). On macOS it defaults to 1 - * to enable metal support, 0 to disable. - * @param mainGPU ??? - * @param lowVRAM ??? - * @param f16KV ??? - * @param logitsAll ??? - * @param vocabOnly ??? - * @param useMMap ??? - * @param useMLock ??? - * @param embeddingOnly ??? - * @param ropeFrequencyBase ??? - * @param ropeFrequencyScale ??? - * @param numThread Sets the number of threads to use during computation. By default, - * Ollama will detect this for optimal performance. It is recommended to set this - * value to the number of physical CPU cores your system has (as opposed to the - * logical number of cores). - * - * Options specified in GenerateRequest. - * @param mirostat Enable Mirostat sampling for controlling perplexity. (default: 0, 0 - * = disabled, 1 = Mirostat, 2 = Mirostat 2.0) - * @param mirostatTau Influences how quickly the algorithm responds to feedback from - * the generated text. A lower learning rate will result in slower adjustments, while - * a higher learning rate will make the algorithm more responsive. (Default: 0.1). - * @param mirostatEta Controls the balance between coherence and diversity of the - * output. A lower value will result in more focused and coherent text. (Default: - * 5.0). - * @param numKeep Unknown. - * @param seed Sets the random number seed to use for generation. Setting this to a - * specific number will make the model generate the same text for the same prompt. - * (Default: 0) - * @param numPredict Maximum number of tokens to predict when generating text. - * (Default: 128, -1 = infinite generation, -2 = fill context) - * @param topK Reduces the probability of generating nonsense. A higher value (e.g. - * 100) will give more diverse answers, while a lower value (e.g. 10) will be more - * conservative. (Default: 40) - * @param topP Works together with top-k. A higher value (e.g., 0.95) will lead to - * more diverse text, while a lower value (e.g., 0.5) will generate more focused and - * conservative text. (Default: 0.9) - * @param tfsZ Tail free sampling is used to reduce the impact of less probable tokens - * from the output. A higher value (e.g., 2.0) will reduce the impact more, while a - * value of 1.0 disables this setting. (default: 1) - * @param typicalP Unknown. - * @param repeatLastN Sets how far back for the model to look back to prevent - * repetition. (Default: 64, 0 = disabled, -1 = num_ctx) - * @param temperature The temperature of the model. Increasing the temperature will - * make the model answer more creatively. (Default: 0.8) - * @param repeatPenalty Sets how strongly to penalize repetitions. A higher value - * (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., - * 0.9) will be more lenient. (Default: 1.1) - * @param presencePenalty Unknown. - * @param frequencyPenalty Unknown. - * @param penalizeNewline Unknown. - * @param stop Sets the stop sequences to use. When this pattern is encountered the - * LLM will stop generating text and return. Multiple stop patterns may be set by - * specifying multiple separate stop parameters in a modelfile. - * @see Ollama - * Valid Parameters and Values - * @see Ollama Go - * Types - */ - @JsonInclude(Include.NON_NULL) - public record Options( - // Runner options which must be set when the model is loaded into memory. - @JsonProperty("numa") Boolean useNUMA, @JsonProperty("num_ctx") Integer numCtx, - @JsonProperty("num_batch") Integer numBatch, @JsonProperty("num_gqa") Integer numGQA, - @JsonProperty("num_gpu") Integer numGPU, @JsonProperty("main_gpu") Integer mainGPU, - @JsonProperty("low_vram") Boolean lowVRAM, @JsonProperty("f16_kv") Boolean f16KV, - @JsonProperty("logits_all") Boolean logitsAll, @JsonProperty("vocab_only") Boolean vocabOnly, - @JsonProperty("use_mmap") Boolean useMMap, @JsonProperty("use_mlock") Boolean useMLock, - @JsonProperty("embedding_only") Boolean embeddingOnly, - @JsonProperty("rope_frequency_base") Float ropeFrequencyBase, - @JsonProperty("rope_frequency_scale") Float ropeFrequencyScale, - @JsonProperty("num_thread") Integer numThread, - - // Options specified in GenerateRequest. - @JsonProperty("num_keep") Integer numKeep, @JsonProperty("seed") Integer seed, - @JsonProperty("num_predict") Integer numPredict, @JsonProperty("top_k") Integer topK, - @JsonProperty("top_p") Float topP, @JsonProperty("tfs_z") Float tfsZ, - @JsonProperty("typical_p") Float typicalP, @JsonProperty("repeat_last_n") Integer repeatLastN, - @JsonProperty("temperature") Float temperature, @JsonProperty("repeat_penalty") Float repeatPenalty, - @JsonProperty("presence_penalty") Float presencePenalty, - @JsonProperty("frequency_penalty") Float frequencyPenalty, @JsonProperty("mirostat") Integer mirostat, - @JsonProperty("mirostat_tau") Float mirostatTau, @JsonProperty("mirostat_eta") Float mirostatEta, - @JsonProperty("penalize_newline") Boolean penalizeNewline, @JsonProperty("stop") String[] stop) { - - /** - * Convert the {@link Options} object to a {@link Map} of key/value pairs. - * @return The {@link Map} of key/value pairs. - */ - public Map toMap() { - try { - var json = new ObjectMapper().writeValueAsString(this); - return new ObjectMapper().readValue(json, new TypeReference>() { - }); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - - private Boolean useNUMA; - - private Integer numCtx; - - private Integer numBatch; - - private Integer numGQA; - - private Integer numGPU; - - private Integer mainGPU; - - private Boolean lowVRAM; - - private Boolean f16KV; - - private Boolean logitsAll; - - private Boolean vocabOnly; - - private Boolean useMMap; - - private Boolean useMLock; - - private Boolean embeddingOnly; - - private Float ropeFrequencyBase; - - private Float ropeFrequencyScale; - - private Integer numThread; - - private Integer numKeep; - - private Integer seed; - - private Integer numPredict; - - private Integer topK; - - private Float topP; - - private Float tfsZ; - - private Float typicalP; - - private Integer repeatLastN; - - private Float temperature; - - private Float repeatPenalty; - - private Float presencePenalty; - - private Float frequencyPenalty; - - private Integer mirostat; - - private Float mirostatTau; - - private Float mirostatEta; - - private Boolean penalizeNewline; - - private String[] stop; - - public Builder withUseNUMA(Boolean useNUMA) { - this.useNUMA = useNUMA; - return this; - } - - public Builder withNumCtx(Integer numCtx) { - this.numCtx = numCtx; - return this; - } - - public Builder withNumBatch(Integer numBatch) { - this.numBatch = numBatch; - return this; - } - - public Builder withNumGQA(Integer numGQA) { - this.numGQA = numGQA; - return this; - } - - public Builder withNumGPU(Integer numGPU) { - this.numGPU = numGPU; - return this; - } - - public Builder withMainGPU(Integer mainGPU) { - this.mainGPU = mainGPU; - return this; - } - - public Builder withLowVRAM(Boolean lowVRAM) { - this.lowVRAM = lowVRAM; - return this; - } - - public Builder withF16KV(Boolean f16KV) { - this.f16KV = f16KV; - return this; - } - - public Builder withLogitsAll(Boolean logitsAll) { - this.logitsAll = logitsAll; - return this; - } - - public Builder withVocabOnly(Boolean vocabOnly) { - this.vocabOnly = vocabOnly; - return this; - } - - public Builder withUseMMap(Boolean useMMap) { - this.useMMap = useMMap; - return this; - } - - public Builder withUseMLock(Boolean useMLock) { - this.useMLock = useMLock; - return this; - } - - public Builder withEmbeddingOnly(Boolean embeddingOnly) { - this.embeddingOnly = embeddingOnly; - return this; - } - - public Builder withRopeFrequencyBase(Float ropeFrequencyBase) { - this.ropeFrequencyBase = ropeFrequencyBase; - return this; - } - - public Builder withRopeFrequencyScale(Float ropeFrequencyScale) { - this.ropeFrequencyScale = ropeFrequencyScale; - return this; - } - - public Builder withNumThread(Integer numThread) { - this.numThread = numThread; - return this; - } - - public Builder withNumKeep(Integer numKeep) { - this.numKeep = numKeep; - return this; - } - - public Builder withSeed(Integer seed) { - this.seed = seed; - return this; - } - - public Builder withNumPredict(Integer numPredict) { - this.numPredict = numPredict; - return this; - } - - public Builder withTopK(Integer topK) { - this.topK = topK; - return this; - } - - public Builder withTopP(Float topP) { - this.topP = topP; - return this; - } - - public Builder withTfsZ(Float tfsZ) { - this.tfsZ = tfsZ; - return this; - } - - public Builder withTypicalP(Float typicalP) { - this.typicalP = typicalP; - return this; - } - - public Builder withRepeatLastN(Integer repeatLastN) { - this.repeatLastN = repeatLastN; - return this; - } - - public Builder withTemperature(Float temperature) { - this.temperature = temperature; - return this; - } - - public Builder withRepeatPenalty(Float repeatPenalty) { - this.repeatPenalty = repeatPenalty; - return this; - } - - public Builder withPresencePenalty(Float presencePenalty) { - this.presencePenalty = presencePenalty; - return this; - } - - public Builder withFrequencyPenalty(Float frequencyPenalty) { - this.frequencyPenalty = frequencyPenalty; - return this; - } - - public Builder withMirostat(Integer mirostat) { - this.mirostat = mirostat; - return this; - } - - public Builder withMirostatTau(Float mirostatTau) { - this.mirostatTau = mirostatTau; - return this; - } - - public Builder withMirostatEta(Float mirostatEta) { - this.mirostatEta = mirostatEta; - return this; - } - - public Builder withPenalizeNewline(Boolean penalizeNewline) { - this.penalizeNewline = penalizeNewline; - return this; - } - - public Builder withStop(String[] stop) { - this.stop = stop; - return this; - } - - public Options build() { - return new Options(useNUMA, numCtx, numBatch, numGQA, numGPU, mainGPU, lowVRAM, f16KV, logitsAll, - vocabOnly, useMMap, useMLock, embeddingOnly, ropeFrequencyBase, ropeFrequencyScale, numThread, - numKeep, seed, numPredict, topK, topP, tfsZ, typicalP, repeatLastN, temperature, repeatPenalty, - presencePenalty, frequencyPenalty, mirostat, mirostatTau, mirostatEta, penalizeNewline, stop); - } - - } - } - -} -// @formatter:on \ No newline at end of file diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java new file mode 100644 index 000000000..172b08fe5 --- /dev/null +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java @@ -0,0 +1,690 @@ +/* + * Copyright 2023-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.ollama.api; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Helper class for creating strongly-typed Ollama options. + * + * @author Christian Tzolov + * @since 0.8.0 + * @see Ollama + * Valid Parameters and Values + * @see Ollama + * Types + */ +@JsonInclude(Include.NON_NULL) +public class OllamaOptions { + + // @formatter:off + /** + * useNUMA Whether to use NUMA. + */ + @JsonProperty("numa") private Boolean useNUMA; + + /** + * Sets the size of the context window used to generate the next token. (Default: 2048) + */ + @JsonProperty("num_ctx") private Integer numCtx; + + /** + * ??? + */ + @JsonProperty("num_batch") private Integer numBatch; + + /** + * The number of GQA groups in the transformer layer. Required for some models, + * for example it is 8 for llama2:70b. + */ + @JsonProperty("num_gqa") private Integer numGQA; + + /** + * The number of layers to send to the GPU(s). On macOS it defaults to 1 + * to enable metal support, 0 to disable. + */ + @JsonProperty("num_gpu") private Integer numGPU; + + /** + * ??? + */ + @JsonProperty("main_gpu")private Integer mainGPU; + + /** + * ??? + */ + @JsonProperty("low_vram") private Boolean lowVRAM; + + /** + * ??? + */ + @JsonProperty("f16_kv") private Boolean f16KV; + + /** + * ??? + */ + @JsonProperty("logits_all") private Boolean logitsAll; + + /** + * ??? + */ + @JsonProperty("vocab_only") private Boolean vocabOnly; + + /** + * ??? + */ + @JsonProperty("use_mmap") private Boolean useMMap; + + /** + * ??? + */ + @JsonProperty("use_mlock") private Boolean useMLock; + + /** + * ??? + */ + @JsonProperty("embedding_only") private Boolean embeddingOnly; + + /** + * ??? + */ + @JsonProperty("rope_frequency_base") private Float ropeFrequencyBase; + + /** + * ??? + */ + @JsonProperty("rope_frequency_scale") private Float ropeFrequencyScale; + + /** + * Sets the number of threads to use during computation. By default, + * Ollama will detect this for optimal performance. It is recommended to set this + * value to the number of physical CPU cores your system has (as opposed to the + * logical number of cores). + */ + @JsonProperty("num_thread") private Integer numThread; + + /** + * ??? + */ + @JsonProperty("num_keep") private Integer numKeep; + + /** + * Sets the random number seed to use for generation. Setting this to a + * specific number will make the model generate the same text for the same prompt. + * (Default: 0) + */ + @JsonProperty("seed") private Integer seed; + + /** + * Maximum number of tokens to predict when generating text. + * (Default: 128, -1 = infinite generation, -2 = fill context) + */ + @JsonProperty("num_predict") private Integer numPredict; + + /** + * Reduces the probability of generating nonsense. A higher value (e.g. + * 100) will give more diverse answers, while a lower value (e.g. 10) will be more + * conservative. (Default: 40) + */ + @JsonProperty("top_k") private Integer topK; + + /** + * Works together with top-k. A higher value (e.g., 0.95) will lead to + * more diverse text, while a lower value (e.g., 0.5) will generate more focused and + * conservative text. (Default: 0.9) + */ + @JsonProperty("top_p") private Float topP; + + /** + * Tail free sampling is used to reduce the impact of less probable tokens + * from the output. A higher value (e.g., 2.0) will reduce the impact more, while a + * value of 1.0 disables this setting. (default: 1) + */ + @JsonProperty("tfs_z") private Float tfsZ; + + /** + * ??? + */ + @JsonProperty("typical_p") private Float typicalP; + + /** + * Sets how far back for the model to look back to prevent + * repetition. (Default: 64, 0 = disabled, -1 = num_ctx) + */ + @JsonProperty("repeat_last_n") private Integer repeatLastN; + + /** + * The temperature of the model. Increasing the temperature will + * make the model answer more creatively. (Default: 0.8) + */ + @JsonProperty("temperature") private Float temperature; + + /** + * Sets how strongly to penalize repetitions. A higher value + * (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., + * 0.9) will be more lenient. (Default: 1.1) + */ + @JsonProperty("repeat_penalty") private Float repeatPenalty; + + /** + * ??? + */ + @JsonProperty("presence_penalty") private Float presencePenalty; + + /** + * ??? + */ + @JsonProperty("frequency_penalty") private Float frequencyPenalty; + + /** + * Enable Mirostat sampling for controlling perplexity. (default: 0, 0 + * = disabled, 1 = Mirostat, 2 = Mirostat 2.0) + */ + @JsonProperty("mirostat") private Integer mirostat; + + /** + * Influences how quickly the algorithm responds to feedback from + * the generated text. A lower learning rate will result in slower adjustments, while + * a higher learning rate will make the algorithm more responsive. (Default: 0.1). + */ + @JsonProperty("mirostat_tau") private Float mirostatTau; + + /** + * Controls the balance between coherence and diversity of the + * output. A lower value will result in more focused and coherent text. (Default: + * 5.0). + */ + @JsonProperty("mirostat_eta") private Float mirostatEta; + + /** + * ??? + */ + @JsonProperty("penalize_newline") private Boolean penalizeNewline; + + /** + * Sets the stop sequences to use. When this pattern is encountered the + * LLM will stop generating text and return. Multiple stop patterns may be set by + * specifying multiple separate stop parameters in a modelfile. + */ + @JsonProperty("stop") private List stop; + + + + public OllamaOptions withUseNUMA(Boolean useNUMA) { + this.useNUMA = useNUMA; + return this; + } + + public OllamaOptions withNumCtx(Integer numCtx) { + this.numCtx = numCtx; + return this; + } + + public OllamaOptions withNumBatch(Integer numBatch) { + this.numBatch = numBatch; + return this; + } + + public OllamaOptions withNumGQA(Integer numGQA) { + this.numGQA = numGQA; + return this; + } + + public OllamaOptions withNumGPU(Integer numGPU) { + this.numGPU = numGPU; + return this; + } + + public OllamaOptions withMainGPU(Integer mainGPU) { + this.mainGPU = mainGPU; + return this; + } + + public OllamaOptions withLowVRAM(Boolean lowVRAM) { + this.lowVRAM = lowVRAM; + return this; + } + + public OllamaOptions withF16KV(Boolean f16KV) { + this.f16KV = f16KV; + return this; + } + + public OllamaOptions withLogitsAll(Boolean logitsAll) { + this.logitsAll = logitsAll; + return this; + } + + public OllamaOptions withVocabOnly(Boolean vocabOnly) { + this.vocabOnly = vocabOnly; + return this; + } + + public OllamaOptions withUseMMap(Boolean useMMap) { + this.useMMap = useMMap; + return this; + } + + public OllamaOptions withUseMLock(Boolean useMLock) { + this.useMLock = useMLock; + return this; + } + + public OllamaOptions withEmbeddingOnly(Boolean embeddingOnly) { + this.embeddingOnly = embeddingOnly; + return this; + } + + public OllamaOptions withRopeFrequencyBase(Float ropeFrequencyBase) { + this.ropeFrequencyBase = ropeFrequencyBase; + return this; + } + + public OllamaOptions withRopeFrequencyScale(Float ropeFrequencyScale) { + this.ropeFrequencyScale = ropeFrequencyScale; + return this; + } + + public OllamaOptions withNumThread(Integer numThread) { + this.numThread = numThread; + return this; + } + + public OllamaOptions withNumKeep(Integer numKeep) { + this.numKeep = numKeep; + return this; + } + + public OllamaOptions withSeed(Integer seed) { + this.seed = seed; + return this; + } + + public OllamaOptions withNumPredict(Integer numPredict) { + this.numPredict = numPredict; + return this; + } + + public OllamaOptions withTopK(Integer topK) { + this.topK = topK; + return this; + } + + public OllamaOptions withTopP(Float topP) { + this.topP = topP; + return this; + } + + public OllamaOptions withTfsZ(Float tfsZ) { + this.tfsZ = tfsZ; + return this; + } + + public OllamaOptions withTypicalP(Float typicalP) { + this.typicalP = typicalP; + return this; + } + + public OllamaOptions withRepeatLastN(Integer repeatLastN) { + this.repeatLastN = repeatLastN; + return this; + } + + public OllamaOptions withTemperature(Float temperature) { + this.temperature = temperature; + return this; + } + + public OllamaOptions withRepeatPenalty(Float repeatPenalty) { + this.repeatPenalty = repeatPenalty; + return this; + } + + public OllamaOptions withPresencePenalty(Float presencePenalty) { + this.presencePenalty = presencePenalty; + return this; + } + + public OllamaOptions withFrequencyPenalty(Float frequencyPenalty) { + this.frequencyPenalty = frequencyPenalty; + return this; + } + + public OllamaOptions withMirostat(Integer mirostat) { + this.mirostat = mirostat; + return this; + } + + public OllamaOptions withMirostatTau(Float mirostatTau) { + this.mirostatTau = mirostatTau; + return this; + } + + public OllamaOptions withMirostatEta(Float mirostatEta) { + this.mirostatEta = mirostatEta; + return this; + } + + public OllamaOptions withPenalizeNewline(Boolean penalizeNewline) { + this.penalizeNewline = penalizeNewline; + return this; + } + + public OllamaOptions withStop(List stop) { + this.stop = stop; + return this; + } + + public Boolean getUseNUMA() { + return useNUMA; + } + + public void setUseNUMA(Boolean useNUMA) { + this.useNUMA = useNUMA; + } + + public Integer getNumCtx() { + return numCtx; + } + + public void setNumCtx(Integer numCtx) { + this.numCtx = numCtx; + } + + public Integer getNumBatch() { + return numBatch; + } + + public void setNumBatch(Integer numBatch) { + this.numBatch = numBatch; + } + + public Integer getNumGQA() { + return numGQA; + } + + public void setNumGQA(Integer numGQA) { + this.numGQA = numGQA; + } + + public Integer getNumGPU() { + return numGPU; + } + + public void setNumGPU(Integer numGPU) { + this.numGPU = numGPU; + } + + public Integer getMainGPU() { + return mainGPU; + } + + public void setMainGPU(Integer mainGPU) { + this.mainGPU = mainGPU; + } + + public Boolean getLowVRAM() { + return lowVRAM; + } + + public void setLowVRAM(Boolean lowVRAM) { + this.lowVRAM = lowVRAM; + } + + public Boolean getF16KV() { + return f16KV; + } + + public void setF16KV(Boolean f16kv) { + f16KV = f16kv; + } + + public Boolean getLogitsAll() { + return logitsAll; + } + + public void setLogitsAll(Boolean logitsAll) { + this.logitsAll = logitsAll; + } + + public Boolean getVocabOnly() { + return vocabOnly; + } + + public void setVocabOnly(Boolean vocabOnly) { + this.vocabOnly = vocabOnly; + } + + public Boolean getUseMMap() { + return useMMap; + } + + public void setUseMMap(Boolean useMMap) { + this.useMMap = useMMap; + } + + public Boolean getUseMLock() { + return useMLock; + } + + public void setUseMLock(Boolean useMLock) { + this.useMLock = useMLock; + } + + public Boolean getEmbeddingOnly() { + return embeddingOnly; + } + + public void setEmbeddingOnly(Boolean embeddingOnly) { + this.embeddingOnly = embeddingOnly; + } + + public Float getRopeFrequencyBase() { + return ropeFrequencyBase; + } + + public void setRopeFrequencyBase(Float ropeFrequencyBase) { + this.ropeFrequencyBase = ropeFrequencyBase; + } + + public Float getRopeFrequencyScale() { + return ropeFrequencyScale; + } + + public void setRopeFrequencyScale(Float ropeFrequencyScale) { + this.ropeFrequencyScale = ropeFrequencyScale; + } + + public Integer getNumThread() { + return numThread; + } + + public void setNumThread(Integer numThread) { + this.numThread = numThread; + } + + public Integer getNumKeep() { + return numKeep; + } + + public void setNumKeep(Integer numKeep) { + this.numKeep = numKeep; + } + + public Integer getSeed() { + return seed; + } + + public void setSeed(Integer seed) { + this.seed = seed; + } + + public Integer getNumPredict() { + return numPredict; + } + + public void setNumPredict(Integer numPredict) { + this.numPredict = numPredict; + } + + public Integer getTopK() { + return topK; + } + + public void setTopK(Integer topK) { + this.topK = topK; + } + + public Float getTopP() { + return topP; + } + + public void setTopP(Float topP) { + this.topP = topP; + } + + public Float getTfsZ() { + return tfsZ; + } + + public void setTfsZ(Float tfsZ) { + this.tfsZ = tfsZ; + } + + public Float getTypicalP() { + return typicalP; + } + + public void setTypicalP(Float typicalP) { + this.typicalP = typicalP; + } + + public Integer getRepeatLastN() { + return repeatLastN; + } + + public void setRepeatLastN(Integer repeatLastN) { + this.repeatLastN = repeatLastN; + } + + public Float getTemperature() { + return temperature; + } + + public void setTemperature(Float temperature) { + this.temperature = temperature; + } + + public Float getRepeatPenalty() { + return repeatPenalty; + } + + public void setRepeatPenalty(Float repeatPenalty) { + this.repeatPenalty = repeatPenalty; + } + + public Float getPresencePenalty() { + return presencePenalty; + } + + public void setPresencePenalty(Float presencePenalty) { + this.presencePenalty = presencePenalty; + } + + public Float getFrequencyPenalty() { + return frequencyPenalty; + } + + public void setFrequencyPenalty(Float frequencyPenalty) { + this.frequencyPenalty = frequencyPenalty; + } + + public Integer getMirostat() { + return mirostat; + } + + public void setMirostat(Integer mirostat) { + this.mirostat = mirostat; + } + + public Float getMirostatTau() { + return mirostatTau; + } + + public void setMirostatTau(Float mirostatTau) { + this.mirostatTau = mirostatTau; + } + + public Float getMirostatEta() { + return mirostatEta; + } + + public void setMirostatEta(Float mirostatEta) { + this.mirostatEta = mirostatEta; + } + + public Boolean getPenalizeNewline() { + return penalizeNewline; + } + + public void setPenalizeNewline(Boolean penalizeNewline) { + this.penalizeNewline = penalizeNewline; + } + + public List getStop() { + return stop; + } + + public void setStop(List stop) { + this.stop = stop; + } + + /** + * Convert the {@link OllamaOptions} object to a {@link Map} of key/value pairs. + * @return The {@link Map} of key/value pairs. + */ + public Map toMap() { + try { + var json = new ObjectMapper().writeValueAsString(this); + return new ObjectMapper().readValue(json, new TypeReference>() { + }); + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Helper factory method to create a new {@link OllamaOptions} instance. + * @return A new {@link OllamaOptions} instance. + */ + public static OllamaOptions create() { + return new OllamaOptions(); + } + + // @formatter:on + +} diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java index c01202207..88a184e6f 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatClientIT.java @@ -18,7 +18,7 @@ import org.testcontainers.junit.jupiter.Testcontainers; import org.springframework.ai.chat.ChatResponse; import org.springframework.ai.chat.Generation; import org.springframework.ai.ollama.api.OllamaApi; -import org.springframework.ai.ollama.api.OllamaApiOptions; +import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.ai.parser.BeanOutputParser; import org.springframework.ai.parser.ListOutputParser; import org.springframework.ai.parser.MapOutputParser; @@ -45,7 +45,7 @@ class OllamaChatClientIT { private static final Log logger = LogFactory.getLog(OllamaChatClientIT.class); @Container - static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.15").withExposedPorts(11434); + static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.16").withExposedPorts(11434); static String baseUrl; @@ -184,7 +184,7 @@ class OllamaChatClientIT { @Bean public OllamaChatClient ollamaChat(OllamaApi ollamaApi) { return new OllamaChatClient(ollamaApi).withModel(MODEL) - .withOptions(OllamaApiOptions.Options.builder().withTemperature(0.9f).build()); + .withOptions(OllamaOptions.create().withTemperature(0.9f)); } } diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java index 095a8f6d4..8016b0c07 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingClientIT.java @@ -31,7 +31,7 @@ class OllamaEmbeddingClientIT { private static final Log logger = LogFactory.getLog(OllamaApiIT.class); @Container - static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.15").withExposedPorts(11434); + static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.16").withExposedPorts(11434); static String baseUrl; diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java index ec60e8a0f..592967ccf 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiIT.java @@ -38,7 +38,6 @@ import org.springframework.ai.ollama.api.OllamaApi.GenerateRequest; import org.springframework.ai.ollama.api.OllamaApi.GenerateResponse; import org.springframework.ai.ollama.api.OllamaApi.Message; import org.springframework.ai.ollama.api.OllamaApi.Message.Role; -import org.springframework.ai.ollama.api.OllamaApiOptions.Options; import static org.assertj.core.api.Assertions.assertThat;; @@ -52,7 +51,7 @@ public class OllamaApiIT { private static final Log logger = LogFactory.getLog(OllamaApiIT.class); @Container - static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.15").withExposedPorts(11434); + static GenericContainer ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.16").withExposedPorts(11434); static OllamaApi ollamaApi; @@ -91,7 +90,7 @@ public class OllamaApiIT { .withMessages(List.of(Message.builder(Role.user) .withContent("What is the capital of Bulgaria and what is the size? " + "What it the national anthem?") .build())) - .withOptions(Options.builder().withTemperature(0.9f).build()) + .withOptions(OllamaOptions.create().withTemperature(0.9f)) .build(); ChatResponse response = ollamaApi.chat(request); @@ -113,7 +112,7 @@ public class OllamaApiIT { .withMessages(List.of(Message.builder(Role.user) .withContent("What is the capital of Bulgaria and what is the size? " + "What it the national anthem?") .build())) - .withOptions(Options.builder().withTemperature(0.9f).build().toMap()) + .withOptions(OllamaOptions.create().withTemperature(0.9f).toMap()) .build(); Flux response = ollamaApi.streamingChat(request); diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaOptionsTests.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaOptionsTests.java new file mode 100644 index 000000000..60c4f098f --- /dev/null +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaOptionsTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2023-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.ollama.api; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +public class OllamaOptionsTests { + + @Test + public void testOptions() { + var options = OllamaOptions.create() + .withTemperature(3.14f) + .withEmbeddingOnly(false) + .withTopK(30) + .withStop(List.of("a", "b", "c")); + + var optionsMap = options.toMap(); + System.out.println(optionsMap); + assertThat(optionsMap).containsEntry("temperature", 3.14); + assertThat(optionsMap).containsEntry("embedding_only", false); + assertThat(optionsMap).containsEntry("top_k", 30); + assertThat(optionsMap).containsEntry("stop", List.of("a", "b", "c")); + } + +} diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/NativeHints.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/NativeHints.java index c76f043da..5952dd031 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/NativeHints.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/NativeHints.java @@ -12,7 +12,7 @@ import org.springframework.ai.bedrock.llama2.api.Llama2ChatBedrockApi; import org.springframework.ai.bedrock.titan.api.TitanChatBedrockApi; import org.springframework.ai.bedrock.titan.api.TitanEmbeddingBedrockApi; import org.springframework.ai.ollama.api.OllamaApi; -import org.springframework.ai.ollama.api.OllamaApiOptions; +import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.ai.vertex.api.VertexAiApi; import org.springframework.aot.hint.MemberCategory; @@ -80,7 +80,7 @@ public class NativeHints implements RuntimeHintsRegistrar { var mcs = MemberCategory.values(); for (var tr : findJsonAnnotatedClasses(OllamaApi.class)) hints.reflection().registerType(tr, mcs); - for (var tr : findJsonAnnotatedClasses(OllamaApiOptions.class)) + for (var tr : findJsonAnnotatedClasses(OllamaOptions.class)) hints.reflection().registerType(tr, mcs); } diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfiguration.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfiguration.java index b2c19c7af..6241f3957 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfiguration.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfiguration.java @@ -18,7 +18,6 @@ package org.springframework.ai.autoconfigure.ollama; import org.springframework.ai.autoconfigure.NativeHints; import org.springframework.ai.ollama.OllamaChatClient; import org.springframework.ai.ollama.api.OllamaApi; -import org.springframework.ai.ollama.api.OllamaApiOptions; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -35,7 +34,7 @@ import org.springframework.context.annotation.ImportRuntimeHints; */ @AutoConfiguration @ConditionalOnClass(OllamaApi.class) -@EnableConfigurationProperties({ OllamaChatProperties.class }) +@EnableConfigurationProperties({ OllamaChatProperties.class, OllamaConnectionProperties.class }) @ConditionalOnProperty(prefix = OllamaChatProperties.CONFIG_PREFIX, name = "enabled", havingValue = "true", matchIfMissing = true) @ImportRuntimeHints(NativeHints.class) @@ -43,32 +42,14 @@ public class OllamaChatAutoConfiguration { @Bean @ConditionalOnMissingBean - public OllamaApi ollamaApi(OllamaChatProperties properties) { + public OllamaApi ollamaApi(OllamaConnectionProperties properties) { return new OllamaApi(properties.getBaseUrl()); } @Bean public OllamaChatClient ollamaChatClient(OllamaApi ollamaApi, OllamaChatProperties properties) { - var optionsBuilder = OllamaApiOptions.Options.builder(); - - if (properties.getTemperature() != null) { - optionsBuilder.withTemperature(properties.getTemperature()); - } - if (properties.getTopK() != null) { - optionsBuilder.withTopK(properties.getTopK()); - } - if (properties.getTopP() != null) { - optionsBuilder.withTopP(properties.getTopP()); - } - - var options = optionsBuilder.build().toMap(); - - if (properties.getOptions() != null) { - options.putAll(properties.getOptions()); - } - - return new OllamaChatClient(ollamaApi).withModel(properties.getModel()).withOptions(options); + return new OllamaChatClient(ollamaApi).withModel(properties.getModel()).withOptions(properties.getOptions()); } } diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatProperties.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatProperties.java index 9be5f861e..626eab54c 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatProperties.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaChatProperties.java @@ -16,8 +16,7 @@ package org.springframework.ai.autoconfigure.ollama; -import java.util.Map; - +import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -31,11 +30,6 @@ public class OllamaChatProperties { public static final String CONFIG_PREFIX = "spring.ai.ollama.chat"; - /** - * Base URL where Ollama API server is running. - */ - private String baseUrl = "http://localhost:11434"; - /** * Enable Ollama Chat Client. True by default. */ @@ -47,35 +41,14 @@ public class OllamaChatProperties { private String model = "llama2"; /** - * (optional) Use a lower value to decrease randomness in the response. Defaults to - * 0.7. + * Client lever Ollama options. Use this property to configure model temperature, topK + * and topP and alike parameters. The null values are ignored defaulting to the + * model's defaults. */ - private Float temperature = 0.8f; - - /** - * (optional) The maximum cumulative probability of tokens to consider when sampling. - * The model uses combined Top-k and nucleus sampling. Nucleus sampling considers the - * smallest set of tokens whose probability sum is at least topP. - */ - private Float topP; - - /** - * Max number or responses to generate. - */ - private Integer topK; - - private Map options; - - public String getBaseUrl() { - return baseUrl; - } - - public void setBaseUrl(String baseUrl) { - this.baseUrl = baseUrl; - } + private OllamaOptions options = new OllamaOptions(); public boolean isEnabled() { - return enabled; + return this.enabled; } public void setEnabled(boolean enabled) { @@ -83,43 +56,15 @@ public class OllamaChatProperties { } public String getModel() { - return model; + return this.model; } public void setModel(String model) { this.model = model; } - public Float getTemperature() { - return temperature; - } - - public void setTemperature(Float temperature) { - this.temperature = temperature; - } - - public Float getTopP() { - return topP; - } - - public void setTopP(Float topP) { - this.topP = topP; - } - - public Integer getTopK() { - return topK; - } - - public void setTopK(Integer maxTokens) { - this.topK = maxTokens; - } - - public void setOptions(Map options) { - this.options = options; - } - - public Map getOptions() { - return options; + public OllamaOptions getOptions() { + return this.options; } } diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaConnectionProperties.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaConnectionProperties.java new file mode 100644 index 000000000..0949567bc --- /dev/null +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaConnectionProperties.java @@ -0,0 +1,45 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.autoconfigure.ollama; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Ollama connection autoconfiguration properties. + * + * @author Christian Tzolov + * @since 0.8.0 + */ +@ConfigurationProperties(OllamaConnectionProperties.CONFIG_PREFIX) +public class OllamaConnectionProperties { + + public static final String CONFIG_PREFIX = "spring.ai.ollama"; + + /** + * Base URL where Ollama API server is running. + */ + private String baseUrl = "http://localhost:11434"; + + public String getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + +} diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfiguration.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfiguration.java index a311a2300..c7600646a 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfiguration.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfiguration.java @@ -35,7 +35,7 @@ import org.springframework.context.annotation.ImportRuntimeHints; */ @AutoConfiguration @ConditionalOnClass(OllamaApi.class) -@EnableConfigurationProperties({ OllamaEmbeddingProperties.class }) +@EnableConfigurationProperties({ OllamaEmbeddingProperties.class, OllamaConnectionProperties.class }) @ConditionalOnProperty(prefix = OllamaEmbeddingProperties.CONFIG_PREFIX, name = "enabled", havingValue = "true", matchIfMissing = true) @ImportRuntimeHints(NativeHints.class) @@ -43,7 +43,7 @@ public class OllamaEmbeddingAutoConfiguration { @Bean @ConditionalOnMissingBean - public OllamaApi ollamaApi(OllamaEmbeddingProperties properties) { + public OllamaApi ollamaApi(OllamaConnectionProperties properties) { return new OllamaApi(properties.getBaseUrl()); } diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingProperties.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingProperties.java index ea64f41bd..5ce77c65d 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingProperties.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingProperties.java @@ -16,8 +16,7 @@ package org.springframework.ai.autoconfigure.ollama; -import java.util.Map; - +import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -36,17 +35,17 @@ public class OllamaEmbeddingProperties { */ private boolean enabled = true; - /** - * Base URL where Ollama API server is running. - */ - private String baseUrl = "http://localhost:11434"; - /** * Ollama Embedding model name. Defaults to 'llama2'. */ private String model = "llama2"; - private Map options; + /** + * Client lever Ollama options. Use this property to configure model temperature, topK + * and topP and alike parameters. The null values are ignored defaulting to the + * model's defaults. + */ + private OllamaOptions options = new OllamaOptions(); public boolean isEnabled() { return enabled; @@ -64,19 +63,7 @@ public class OllamaEmbeddingProperties { this.model = model; } - public String getBaseUrl() { - return baseUrl; - } - - public void setBaseUrl(String baseUrl) { - this.baseUrl = baseUrl; - } - - public void setOptions(Map clientOptions) { - this.options = clientOptions; - } - - public Map getOptions() { + public OllamaOptions getOptions() { return options; } diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfigurationIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfigurationIT.java index f4387f185..87bc06ab8 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfigurationIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfigurationIT.java @@ -71,8 +71,8 @@ public class OllamaChatAutoConfigurationIT { private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() .withPropertyValues("spring.ai.ollama.chat.enabled=true", "spring.ai.ollama.chat.model=" + MODEL_NAME, - "spring.ai.ollama.chat.baseUrl=" + baseUrl, "spring.ai.ollama.chat.temperature=0.5", - "spring.ai.ollama.chat.topK=500") + "spring.ai.ollama.baseUrl=" + baseUrl, "spring.ai.ollama.chat.temperature=0.5", + "spring.ai.ollama.chat.topK=10") .withConfiguration(AutoConfigurations.of(OllamaChatAutoConfiguration.class)); private final Message systemMessage = new SystemPromptTemplate(""" diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfigurationTests.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfigurationTests.java index 0dc856d75..573e30b2d 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfigurationTests.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaChatAutoConfigurationTests.java @@ -33,21 +33,29 @@ public class OllamaChatAutoConfigurationTests { @Test public void propertiesTest() { - new ApplicationContextRunner() - .withPropertyValues("spring.ai.ollama.chat.enabled=true", "spring.ai.ollama.chat.model=MODEL_XYZ", - "spring.ai.ollama.chat.temperature=0.55", "spring.ai.ollama.chat.topP=0.55", - "spring.ai.ollama.chat.topK=123") + new ApplicationContextRunner().withPropertyValues( + // @formatter:off + "spring.ai.ollama.base-url=TEST_BASE_URL", + "spring.ai.ollama.chat.enabled=true", + "spring.ai.ollama.chat.model=MODEL_XYZ", + "spring.ai.ollama.chat.options.temperature=0.55", + "spring.ai.ollama.chat.options.topP=0.56", + "spring.ai.ollama.chat.options.topK=123") + // @formatter:on .withConfiguration(AutoConfigurations.of(OllamaChatAutoConfiguration.class)) .run(context -> { var chatProperties = context.getBean(OllamaChatProperties.class); + var connectionProperties = context.getBean(OllamaConnectionProperties.class); + + assertThat(connectionProperties.getBaseUrl()).isEqualTo("TEST_BASE_URL"); assertThat(chatProperties.isEnabled()).isTrue(); assertThat(chatProperties.getModel()).isEqualTo("MODEL_XYZ"); - assertThat(chatProperties.getTemperature()).isEqualTo(0.55f); - assertThat(chatProperties.getTopP()).isEqualTo(0.55f); + assertThat(chatProperties.getOptions().getTemperature()).isEqualTo(0.55f); + assertThat(chatProperties.getOptions().getTopP()).isEqualTo(0.56f); - assertThat(chatProperties.getTopK()).isEqualTo(123); + assertThat(chatProperties.getOptions().getTopK()).isEqualTo(123); }); } diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java index 3731ac2dc..c23d7308d 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationIT.java @@ -63,7 +63,7 @@ public class OllamaEmbeddingAutoConfigurationIT { private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() .withPropertyValues("spring.ai.ollama.embedding.enabled=true", "spring.ai.ollama.embedding.model=" + MODEL_NAME, - "spring.ai.ollama.embedding.base-url=" + baseUrl) + "spring.ai.ollama.base-url=" + baseUrl) .withConfiguration(AutoConfigurations.of(OllamaEmbeddingAutoConfiguration.class)); @Test diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationTests.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationTests.java index e4c3d0295..46c0f05f9 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationTests.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaEmbeddingAutoConfigurationTests.java @@ -33,21 +33,23 @@ public class OllamaEmbeddingAutoConfigurationTests { @Test public void propertiesTest() { - new ApplicationContextRunner().withPropertyValues("spring.ai.ollama.embedding.enabled=true", - "spring.ai.ollama.embedding.base-url=TEST_BASE_URL", "spring.ai.ollama.embedding.model=MODEL_XYZ", - "spring.ai.ollama.embedding.options.temperature=0.13" // TODO: Fix the - // float parsing - ).withConfiguration(AutoConfigurations.of(OllamaEmbeddingAutoConfiguration.class)).run(context -> { - var properties = context.getBean(OllamaEmbeddingProperties.class); + new ApplicationContextRunner() + .withPropertyValues("spring.ai.ollama.base-url=TEST_BASE_URL", "spring.ai.ollama.embedding.enabled=true", + "spring.ai.ollama.embedding.model=MODEL_XYZ", "spring.ai.ollama.embedding.options.temperature=0.13", + "spring.ai.ollama.embedding.options.topK=13") + .withConfiguration(AutoConfigurations.of(OllamaEmbeddingAutoConfiguration.class)) + .run(context -> { + var embeddingProperties = context.getBean(OllamaEmbeddingProperties.class); + var connectionProperties = context.getBean(OllamaConnectionProperties.class); - // java.lang.Float.valueOf(0.13f) - assertThat(properties.isEnabled()).isTrue(); - assertThat(properties.getModel()).isEqualTo("MODEL_XYZ"); - assertThat(properties.getBaseUrl()).isEqualTo("TEST_BASE_URL"); - assertThat(properties.getOptions()).containsKeys("temperature"); - assertThat(properties.getOptions().get("temperature")).isEqualTo("0.13"); - - }); + // java.lang.Float.valueOf(0.13f) + assertThat(embeddingProperties.isEnabled()).isTrue(); + assertThat(embeddingProperties.getModel()).isEqualTo("MODEL_XYZ"); + assertThat(connectionProperties.getBaseUrl()).isEqualTo("TEST_BASE_URL"); + assertThat(embeddingProperties.getOptions().toMap()).containsKeys("temperature"); + assertThat(embeddingProperties.getOptions().toMap().get("temperature")).isEqualTo(0.13); + assertThat(embeddingProperties.getOptions().getTopK()).isEqualTo(13); + }); } @Test