From d5bc9c998cf70917da327c0e2aae5b5a9308979a Mon Sep 17 00:00:00 2001 From: Thomas Vitale Date: Mon, 21 Oct 2024 19:56:01 +0200 Subject: [PATCH] Consolidate Ollama auto-pull logic Consolidate the Ollama auto-pull logic at startup time, supporting the auto-pull for the default models specified via configuration properties and for optional models specified for initialization. Signed-off-by: Thomas Vitale --- .../ai/ollama/OllamaChatModel.java | 11 +-- .../ai/ollama/OllamaEmbeddingModel.java | 11 +-- .../ai/ollama/api/OllamaOptions.java | 28 +------- .../management/ModelManagementOptions.java | 42 +++++++++++ .../ai/ollama/OllamaChatModelIT.java | 19 ++--- .../ai/ollama/OllamaEmbeddingModelIT.java | 24 +++---- .../ai/ollama/OllamaImage.java | 2 +- .../ai/ollama/api/OllamaApiModelsIT.java | 14 ++-- .../ROOT/pages/api/chat/comparison.adoc | 3 +- .../ROOT/pages/api/chat/ollama-chat.adoc | 70 ++++++++----------- .../api/embeddings/ollama-embeddings.adoc | 67 ++++++++---------- .../ai/autoconfigure/ollama/OllamaImage.java | 2 +- .../connection/ollama/OllamaImage.java | 2 +- 13 files changed, 140 insertions(+), 155 deletions(-) diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java index 2e9532461..93edabe8c 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java @@ -97,7 +97,7 @@ public class OllamaChatModel extends AbstractToolCallSupport implements ChatMode this.defaultOptions = defaultOptions; this.observationRegistry = observationRegistry; this.modelManager = new OllamaModelManager(chatApi, modelManagementOptions); - initializeModelIfEnabled(defaultOptions.getModel(), modelManagementOptions.pullModelStrategy()); + initializeModel(defaultOptions.getModel(), modelManagementOptions.pullModelStrategy()); } public static Builder builder() { @@ -302,11 +302,6 @@ public class OllamaChatModel extends AbstractToolCallSupport implements ChatMode } OllamaOptions mergedOptions = ModelOptionsUtils.merge(runtimeOptions, this.defaultOptions, OllamaOptions.class); - mergedOptions.setPullModelStrategy(this.defaultOptions.getPullModelStrategy()); - if (runtimeOptions != null && runtimeOptions.getPullModelStrategy() != null) { - mergedOptions.setPullModelStrategy(runtimeOptions.getPullModelStrategy()); - } - // Override the model. if (!StringUtils.hasText(mergedOptions.getModel())) { throw new IllegalArgumentException("Model is not set!"); @@ -331,8 +326,6 @@ public class OllamaChatModel extends AbstractToolCallSupport implements ChatMode requestBuilder.withTools(this.getFunctionTools(functionsForThisRequest)); } - initializeModelIfEnabled(mergedOptions.getModel(), mergedOptions.getPullModelStrategy()); - return requestBuilder.build(); } @@ -379,7 +372,7 @@ public class OllamaChatModel extends AbstractToolCallSupport implements ChatMode /** * Pull the given model into Ollama based on the specified strategy. */ - private void initializeModelIfEnabled(String model, PullModelStrategy pullModelStrategy) { + private void initializeModel(String model, PullModelStrategy pullModelStrategy) { if (pullModelStrategy != null && !PullModelStrategy.NEVER.equals(pullModelStrategy)) { this.modelManager.pullModel(model, pullModelStrategy); } diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingModel.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingModel.java index 534d1ba33..7034a9c03 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingModel.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaEmbeddingModel.java @@ -77,7 +77,7 @@ public class OllamaEmbeddingModel extends AbstractEmbeddingModel { this.observationRegistry = observationRegistry; this.modelManager = new OllamaModelManager(ollamaApi, modelManagementOptions); - initializeModelIfEnabled(defaultOptions.getModel(), modelManagementOptions.pullModelStrategy()); + initializeModel(defaultOptions.getModel(), modelManagementOptions.pullModelStrategy()); } public static Builder builder() { @@ -139,19 +139,12 @@ public class OllamaEmbeddingModel extends AbstractEmbeddingModel { OllamaOptions mergedOptions = ModelOptionsUtils.merge(runtimeOptions, this.defaultOptions, OllamaOptions.class); - mergedOptions.setPullModelStrategy(this.defaultOptions.getPullModelStrategy()); - if (runtimeOptions != null && runtimeOptions.getPullModelStrategy() != null) { - mergedOptions.setPullModelStrategy(runtimeOptions.getPullModelStrategy()); - } - // Override the model. if (!StringUtils.hasText(mergedOptions.getModel())) { throw new IllegalArgumentException("Model is not set!"); } String model = mergedOptions.getModel(); - initializeModelIfEnabled(mergedOptions.getModel(), mergedOptions.getPullModelStrategy()); - return new OllamaApi.EmbeddingsRequest(model, inputContent, DurationParser.parse(mergedOptions.getKeepAlive()), OllamaOptions.filterNonSupportedFields(mergedOptions.toMap()), mergedOptions.getTruncate()); } @@ -163,7 +156,7 @@ public class OllamaEmbeddingModel extends AbstractEmbeddingModel { /** * Pull the given model into Ollama based on the specified strategy. */ - private void initializeModelIfEnabled(String model, PullModelStrategy pullModelStrategy) { + private void initializeModel(String model, PullModelStrategy pullModelStrategy) { if (pullModelStrategy != null && !PullModelStrategy.NEVER.equals(pullModelStrategy)) { this.modelManager.pullModel(model, pullModelStrategy); } diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java index f7a780668..a0dad31a4 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/api/OllamaOptions.java @@ -28,7 +28,6 @@ import org.springframework.ai.embedding.EmbeddingOptions; import org.springframework.ai.model.ModelOptionsUtils; import org.springframework.ai.model.function.FunctionCallback; import org.springframework.ai.model.function.FunctionCallingOptions; -import org.springframework.ai.ollama.management.PullModelStrategy; import org.springframework.boot.context.properties.NestedConfigurationProperty; import org.springframework.util.Assert; @@ -303,12 +302,6 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed @JsonIgnore private Map toolContext; - /** - * Strategy for pulling models at run-time. - */ - @JsonIgnore - private PullModelStrategy pullModelStrategy; - public static OllamaOptions builder() { return new OllamaOptions(); } @@ -521,11 +514,6 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed return this; } - public OllamaOptions withPullModelStrategy(PullModelStrategy pullModelStrategy) { - this.pullModelStrategy = pullModelStrategy; - return this; - } - // ------------------- // Getters and Setters // ------------------- @@ -866,14 +854,6 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed this.toolContext = toolContext; } - public PullModelStrategy getPullModelStrategy() { - return this.pullModelStrategy; - } - - public void setPullModelStrategy(PullModelStrategy pullModelStrategy) { - this.pullModelStrategy = pullModelStrategy; - } - /** * Convert the {@link OllamaOptions} object to a {@link Map} of key/value pairs. * @return The {@link Map} of key/value pairs. @@ -944,8 +924,7 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed .withFunctions(fromOptions.getFunctions()) .withProxyToolCalls(fromOptions.getProxyToolCalls()) .withFunctionCallbacks(fromOptions.getFunctionCallbacks()) - .withToolContext(fromOptions.getToolContext()) - .withPullModelStrategy(fromOptions.getPullModelStrategy()); + .withToolContext(fromOptions.getToolContext()); } // @formatter:on @@ -975,8 +954,7 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed && Objects.equals(penalizeNewline, that.penalizeNewline) && Objects.equals(stop, that.stop) && Objects.equals(functionCallbacks, that.functionCallbacks) && Objects.equals(proxyToolCalls, that.proxyToolCalls) && Objects.equals(functions, that.functions) - && Objects.equals(toolContext, that.toolContext) - && Objects.equals(pullModelStrategy, that.pullModelStrategy); + && Objects.equals(toolContext, that.toolContext); } @Override @@ -987,7 +965,7 @@ public class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed this.topP, tfsZ, this.typicalP, this.repeatLastN, this.temperature, this.repeatPenalty, this.presencePenalty, this.frequencyPenalty, this.mirostat, this.mirostatTau, this.mirostatEta, this.penalizeNewline, this.stop, this.functionCallbacks, this.functions, this.proxyToolCalls, - this.toolContext, this.pullModelStrategy); + this.toolContext); } } diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/management/ModelManagementOptions.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/management/ModelManagementOptions.java index b49d0978c..5d600b14e 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/management/ModelManagementOptions.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/management/ModelManagementOptions.java @@ -26,7 +26,49 @@ import java.util.List; */ public record ModelManagementOptions(PullModelStrategy pullModelStrategy, List additionalModels, Duration timeout, Integer maxRetries) { + public static ModelManagementOptions defaults() { return new ModelManagementOptions(PullModelStrategy.NEVER, List.of(), Duration.ofMinutes(5), 0); } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private PullModelStrategy pullModelStrategy = PullModelStrategy.NEVER; + + private List additionalModels = List.of(); + + private Duration timeout = Duration.ofMinutes(5); + + private Integer maxRetries = 0; + + public Builder withPullModelStrategy(PullModelStrategy pullModelStrategy) { + this.pullModelStrategy = pullModelStrategy; + return this; + } + + public Builder withAdditionalModels(List additionalModels) { + this.additionalModels = additionalModels; + return this; + } + + public Builder withTimeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + public Builder withMaxRetries(Integer maxRetries) { + this.maxRetries = maxRetries; + return this; + } + + public ModelManagementOptions build() { + return new ModelManagementOptions(pullModelStrategy, additionalModels, timeout, maxRetries); + } + + } + } diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelIT.java index 3b23f39b1..4b2fac29e 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelIT.java @@ -33,6 +33,7 @@ import org.springframework.ai.converter.ListOutputConverter; import org.springframework.ai.converter.MapOutputConverter; import org.springframework.ai.ollama.api.OllamaApi; import org.springframework.ai.ollama.api.OllamaModel; +import org.springframework.ai.ollama.management.ModelManagementOptions; import org.springframework.ai.ollama.management.OllamaModelManager; import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.ai.ollama.management.PullModelStrategy; @@ -56,6 +57,8 @@ class OllamaChatModelIT extends BaseOllamaIT { private static final String MODEL = OllamaModel.LLAMA3_2.getName(); + private static final String ADDITIONAL_MODEL = "tinyllama"; + @Autowired private OllamaChatModel chatModel; @@ -65,23 +68,17 @@ class OllamaChatModelIT extends BaseOllamaIT { @Test void autoPullModelTest() { var modelManager = new OllamaModelManager(ollamaApi); - var model = "tinyllama"; - modelManager.deleteModel(model); - assertThat(modelManager.isModelAvailable(model)).isFalse(); + assertThat(modelManager.isModelAvailable(ADDITIONAL_MODEL)).isTrue(); String joke = ChatClient.create(chatModel) .prompt("Tell me a joke") - .options(OllamaOptions.builder() - .withModel(model) - .withPullModelStrategy(PullModelStrategy.WHEN_MISSING) - .build()) + .options(OllamaOptions.builder().withModel(ADDITIONAL_MODEL).build()) .call() .content(); assertThat(joke).isNotEmpty(); - assertThat(modelManager.isModelAvailable(model)).isTrue(); - modelManager.deleteModel(model); + modelManager.deleteModel(ADDITIONAL_MODEL); } @Test @@ -249,6 +246,10 @@ class OllamaChatModelIT extends BaseOllamaIT { return OllamaChatModel.builder() .withOllamaApi(ollamaApi) .withDefaultOptions(OllamaOptions.create().withModel(MODEL).withTemperature(0.9)) + .withModelManagementOptions(ModelManagementOptions.builder() + .withPullModelStrategy(PullModelStrategy.WHEN_MISSING) + .withAdditionalModels(List.of(ADDITIONAL_MODEL)) + .build()) .build(); } diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingModelIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingModelIT.java index f7ce804e0..ae0612ac3 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingModelIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaEmbeddingModelIT.java @@ -21,6 +21,7 @@ import org.springframework.ai.embedding.EmbeddingRequest; import org.springframework.ai.embedding.EmbeddingResponse; import org.springframework.ai.ollama.api.OllamaApi; import org.springframework.ai.ollama.api.OllamaModel; +import org.springframework.ai.ollama.management.ModelManagementOptions; import org.springframework.ai.ollama.management.OllamaModelManager; import org.springframework.ai.ollama.api.OllamaOptions; import org.springframework.ai.ollama.management.PullModelStrategy; @@ -41,6 +42,8 @@ class OllamaEmbeddingModelIT extends BaseOllamaIT { private static final String MODEL = OllamaModel.NOMIC_EMBED_TEXT.getName(); + private static final String ADDITIONAL_MODEL = "all-minilm"; + @Autowired private OllamaEmbeddingModel embeddingModel; @@ -65,36 +68,29 @@ class OllamaEmbeddingModelIT extends BaseOllamaIT { } @Test - void autoPullModel() { + void autoPullModelAtStartupTime() { var model = "all-minilm"; assertThat(embeddingModel).isNotNull(); var modelManager = new OllamaModelManager(ollamaApi); - modelManager.deleteModel(model); - assertThat(modelManager.isModelAvailable(model)).isFalse(); + assertThat(modelManager.isModelAvailable(ADDITIONAL_MODEL)).isTrue(); EmbeddingResponse embeddingResponse = embeddingModel .call(new EmbeddingRequest(List.of("Hello World", "Something else"), - OllamaOptions.builder() - .withModel(model) - .withPullModelStrategy(PullModelStrategy.WHEN_MISSING) - .withTruncate(false) - .build())); - - assertThat(modelManager.isModelAvailable(model)).isTrue(); + OllamaOptions.builder().withModel(model).withTruncate(false).build())); assertThat(embeddingResponse.getResults()).hasSize(2); assertThat(embeddingResponse.getResults().get(0).getIndex()).isEqualTo(0); assertThat(embeddingResponse.getResults().get(0).getOutput()).isNotEmpty(); assertThat(embeddingResponse.getResults().get(1).getIndex()).isEqualTo(1); assertThat(embeddingResponse.getResults().get(1).getOutput()).isNotEmpty(); - assertThat(embeddingResponse.getMetadata().getModel()).contains(model); + assertThat(embeddingResponse.getMetadata().getModel()).contains(ADDITIONAL_MODEL); assertThat(embeddingResponse.getMetadata().getUsage().getPromptTokens()).isEqualTo(4); assertThat(embeddingResponse.getMetadata().getUsage().getTotalTokens()).isEqualTo(4); assertThat(embeddingModel.dimensions()).isEqualTo(768); - modelManager.deleteModel(model); + modelManager.deleteModel(ADDITIONAL_MODEL); } @SpringBootConfiguration @@ -110,6 +106,10 @@ class OllamaEmbeddingModelIT extends BaseOllamaIT { return OllamaEmbeddingModel.builder() .withOllamaApi(ollamaApi) .withDefaultOptions(OllamaOptions.create().withModel(MODEL)) + .withModelManagementOptions(ModelManagementOptions.builder() + .withPullModelStrategy(PullModelStrategy.WHEN_MISSING) + .withAdditionalModels(List.of(ADDITIONAL_MODEL)) + .build()) .build(); } diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaImage.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaImage.java index 0b57cb16e..8a13c29b5 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaImage.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaImage.java @@ -22,6 +22,6 @@ import org.testcontainers.utility.DockerImageName; */ public class OllamaImage { - public static final DockerImageName DEFAULT_IMAGE = DockerImageName.parse("ollama/ollama:0.3.13"); + public static final DockerImageName DEFAULT_IMAGE = DockerImageName.parse("ollama/ollama:0.3.14"); } diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiModelsIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiModelsIT.java index c6a7c67e1..bc4e878ab 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiModelsIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/api/OllamaApiModelsIT.java @@ -15,6 +15,11 @@ */ package org.springframework.ai.ollama.api; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.time.Duration; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledIf; @@ -22,11 +27,6 @@ import org.springframework.ai.ollama.BaseOllamaIT; import org.springframework.http.HttpStatus; import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.IOException; -import java.time.Duration; - -import static org.assertj.core.api.Assertions.assertThat; - /** * Integration tests for the Ollama APIs to manage models. * @@ -36,7 +36,7 @@ import static org.assertj.core.api.Assertions.assertThat; @DisabledIf("isDisabled") public class OllamaApiModelsIT extends BaseOllamaIT { - private static final String MODEL = OllamaModel.NOMIC_EMBED_TEXT.getName(); + private static final String MODEL = "all-minilm"; static OllamaApi ollamaApi; @@ -60,7 +60,7 @@ public class OllamaApiModelsIT extends BaseOllamaIT { var showModelResponse = ollamaApi.showModel(showModelRequest); assertThat(showModelResponse).isNotNull(); - assertThat(showModelResponse.details().family()).isEqualTo("nomic-bert"); + assertThat(showModelResponse.details().family()).isEqualTo("bert"); } @Test diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc index 67555dfe9..b9a864dba 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc @@ -2,7 +2,7 @@ // :YES: image::yes.svg[width=16] // :NO: image::no.svg[width=12] -// [%autowidth] + This table compares various Chat Models supported by Spring AI, detailing their capabilities: @@ -39,6 +39,5 @@ This table compares various Chat Models supported by Spring AI, detailing their | xref::api/chat/bedrock/bedrock-llama.adoc[Amazon Bedrock/Llama] | text ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] | xref::api/chat/bedrock/bedrock-titan.adoc[Amazon Bedrock/Titan] | text ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] | xref::api/chat/bedrock/bedrock-anthropic3.adoc[Amazon Bedrock/Anthropic 3] | text ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] - |==== diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc index 76f4d188e..71e30665b 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/ollama-chat.adoc @@ -95,7 +95,6 @@ Here are the advanced request parameter for the Ollama chat model: | spring.ai.ollama.chat.options.model | The name of the https://github.com/ollama/ollama?tab=readme-ov-file#model-library[supported model] to use. | mistral | spring.ai.ollama.chat.options.format | The format to return a response in. Currently, the only accepted value is `json` | - | spring.ai.ollama.chat.options.keep_alive | Controls how long the model will stay loaded into memory following the request | 5m -| spring.ai.ollama.chat.options.pull-model-strategy | Strategy for pulling models at run-time. | `never` |==== The remaining `options` properties are based on the link:https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values[Ollama Valid Parameters and Values] and link:https://github.com/ollama/ollama/blob/main/api/types.go[Ollama Types]. The default values are based on the link:https://github.com/ollama/ollama/blob/b538dc3858014f94b099730a592751a5454cab0a/api/types.go#L364[Ollama Types Defaults]. @@ -164,23 +163,21 @@ TIP: In addition to the model specific link:https://github.com/spring-projects/s [[auto-pulling-models]] == Auto-pulling Models -Spring AI Ollama can automatically pull models when not available in your Ollama instance. -This feature is particularly useful when working with different models or when deploying your application to new environments. +Spring AI Ollama can automatically pull models when they are not available in your Ollama instance. +This feature is particularly useful for development and testing as well as for deploying your applications to new environments. -TIP: you can also pull, by name, any of the thousands, free, xref:https://huggingface.co/models?library=gguf&sort=trending[GGUF Hugging Face Models]. +TIP: You can also pull, by name, any of the thousands, free, xref:https://huggingface.co/models?library=gguf&sort=trending[GGUF Hugging Face Models]. There are three strategies for pulling models: -* `always` (defined in `PullModelStrategy.ALWAYS`). Always pull the model, even if it's already available. Useful to ensure you're using the latest version of that model. -* `when_missing` (defined in `PullModelStrategy.WHEN_MISSING`). Only pull the model if it's not already available. It might be an older version of the model. -* `never` (defined in `PullModelStrategy.NEVER`). Never pull the model. +* `always` (defined in `PullModelStrategy.ALWAYS`): Always pull the model, even if it's already available. Useful to ensure you're using the latest version of the model. +* `when_missing` (defined in `PullModelStrategy.WHEN_MISSING`): Only pull the model if it's not already available. This may result in using an older version of the model. +* `never` (defined in `PullModelStrategy.NEVER`): Never pull the model automatically. -CAUTION: Due to the unexpected delays while downloading models, this feature is not recommended for production environments. Instead, consider to assess and pre-download the necessary models in advance. - -=== Pulling models at startup time +CAUTION: Due to potential delays while downloading models, automatic pulling is not recommended for production environments. Instead, consider assessing and pre-downloading the necessary models in advance. All models defined via configuration properties and default options can be automatically pulled at startup time. -You can configure strategy, timeout, and max number of retries via configuration properties. +You can configure the pull strategy, timeout, and maximum number of retries using configuration properties: [source,yaml] ---- @@ -193,9 +190,9 @@ spring: max-retries: 1 ---- -CAUTION: The application will not complete its initialization until all the models become available in Ollama. Depending on the model size and the speed of the Internet connection, your application might be slow at starting up. +CAUTION: The application will not complete its initialization until all specified models are available in Ollama. Depending on the model size and internet connection speed, this may significantly slow down your application's startup time. -You can also initialize additional models at startup time, useful for those models used dynamically at runtime. +You can initialize additional models at startup, which is useful for models used dynamically at runtime: [source,yaml] ---- @@ -210,7 +207,7 @@ spring: - qwen2.5 ---- -If you want to apply the pulling strategy only to other types of models, you can exclude the chat models from the initialization task. +If you want to apply the pulling strategy only to specific types of models, you can exclude chat models from the initialization task: [source,yaml] ---- @@ -223,24 +220,7 @@ spring: include: false ---- -=== Pulling models at runtime - -To enable auto-pulling of models at runtime, you can configure the `pullModelStrategy` option in your `OllamaOptions`: - -[source,java] ----- -ChatResponse response = chatModel.call(new Prompt( - "Generate the names of 5 famous pirates.", - OllamaOptions.builder() - .withModel("llama3.2") - .withPullModelStrategy(PullModelStrategy.ALWAYS) - .build() - )); ----- - -You can also configure this option using the following property: `spring.ai.ollama.chat.options.pull-model-strategy=always`. - -CAUTION: The time to process an incoming request might incur unexpected delays, waiting for the needed model to become available in Ollama. Depending on the model size and the speed of the Internet connection, your application might be slow at processing requests. You might want to initialize these models at startup time instead, using the `spring.ai.ollama.init.chat.additional-models` property. +This configuration will apply the pulling strategy to all models except chat models. == Function Calling @@ -304,13 +284,18 @@ Check the link:https://github.com/spring-projects/spring-ai/blob/main/models/spr https://start.spring.io/[Create] a new Spring Boot project and add the `spring-ai-ollama-spring-boot-starter` to your pom (or gradle) dependencies. -Add a `application.properties` file, under the `src/main/resources` directory, to enable and configure the Ollama chat model: +Add a `application.yaml` file, under the `src/main/resources` directory, to enable and configure the Ollama chat model: -[source,application.properties] +[source,yaml] ---- -spring.ai.ollama.base-url=http://localhost:11434 -spring.ai.ollama.chat.options.model=mistral -spring.ai.ollama.chat.options.temperature=0.7 +spring: + ai: + ollama: + base-url: http://localhost:11434 + chat: + options: + model: mistral + temperature: 0.7 ---- TIP: Replace the `base-url` with your Ollama server URL. @@ -349,8 +334,12 @@ public class ChatController { If you don't want to use the Spring Boot auto-configuration, you can manually configure the `OllamaChatModel` in your application. The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java[OllamaChatModel] implements the `ChatModel` and `StreamingChatModel` and uses the <> to connect to the Ollama service. -To use it, add the `spring-ai-ollama` dependency to your project's Maven `pom.xml` file: +To use it, add the `spring-ai-ollama` dependency to your project's Maven `pom.xml` or Gradle `build.gradle` build files: +[tabs] +====== +Maven:: ++ [source,xml] ---- @@ -359,14 +348,15 @@ To use it, add the `spring-ai-ollama` dependency to your project's Maven `pom.xm ---- -or to your Gradle `build.gradle` build file. - +Gradle:: ++ [source,groovy] ---- dependencies { implementation 'org.springframework.ai:spring-ai-ollama' } ---- +====== TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file. diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/embeddings/ollama-embeddings.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/embeddings/ollama-embeddings.adoc index e414df625..726367892 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/embeddings/ollama-embeddings.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/embeddings/ollama-embeddings.adoc @@ -7,8 +7,6 @@ Small distances suggest high relatedness and large distances suggest low related The `OllamaEmbeddingModel` implementation leverages the Ollama https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings[Embeddings API] endpoint. -TIP: you can also pull, by name, any of the thousands, free, xref:https://huggingface.co/models?library=gguf&sort=trending[GGUF HuggingFace Models] - == Prerequisites You first need access to an Ollama instance. There are a few options, including the following: @@ -36,8 +34,12 @@ Alternatively, you can enable the option to download automatically any needed mo == Auto-configuration Spring AI provides Spring Boot auto-configuration for the Azure Ollama Embedding Model. -To enable it add the following dependency to your Maven `pom.xml` file: +To enable it add the following dependency to your Maven `pom.xml` or Gradle `build.gradle` build files: +[tabs] +====== +Maven:: ++ [source,xml] ---- @@ -46,14 +48,15 @@ To enable it add the following dependency to your Maven `pom.xml` file: ---- -or to your Gradle `build.gradle` build file. - +Gradle:: ++ [source,groovy] ---- dependencies { implementation 'org.springframework.ai:spring-ai-ollama-spring-boot-starter' } ---- +====== TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file. Spring AI artifacts are published in Spring Milestone and Snapshot repositories. Refer to the Repositories section to add these repositories to your build system. @@ -95,7 +98,6 @@ Here are the advanced request parameter for the Ollama embedding model: You can use dedicated https://ollama.com/search?c=embedding[Embedding Model] types | mistral | spring.ai.ollama.embedding.options.keep_alive | Controls how long the model will stay loaded into memory following the request | 5m | spring.ai.ollama.embedding.options.truncate | Truncates the end of each input to fit within context length. Returns error if false and context length is exceeded. | true -| spring.ai.ollama.embedding.options.pull-model-strategy | Strategy for pulling models at run-time. | `never` |==== The remaining `options` properties are based on the link:https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values[Ollama Valid Parameters and Values] and link:https://github.com/ollama/ollama/blob/main/api/types.go[Ollama Types]. The default values are based on: link:https://github.com/ollama/ollama/blob/b538dc3858014f94b099730a592751a5454cab0a/api/types.go#L364[Ollama type defaults]. @@ -161,23 +163,21 @@ EmbeddingResponse embeddingResponse = embeddingModel.call( [[auto-pulling-models]] == Auto-pulling Models -Spring AI Ollama can automatically pull models when not available in your Ollama instance. -This feature is particularly useful when working with different models or when deploying your application to new environments. +Spring AI Ollama can automatically pull models when they are not available in your Ollama instance. +This feature is particularly useful for development and testing as well as for deploying your applications to new environments. -TIP: you can also pull, by name, any of the thousands, free, xref:https://huggingface.co/models?library=gguf&sort=trending[GGUF Hugging Face Models]. +TIP: You can also pull, by name, any of the thousands, free, xref:https://huggingface.co/models?library=gguf&sort=trending[GGUF Hugging Face Models]. There are three strategies for pulling models: -* `always` (defined in `PullModelStrategy.ALWAYS`). Always pull the model, even if it's already available. Useful to ensure you're using the latest version of that model. -* `when_missing` (defined in `PullModelStrategy.WHEN_MISSING`). Only pull the model if it's not already available. It might be an older version of the model. -* `never` (defined in `PullModelStrategy.NEVER`). Never pull the model. +* `always` (defined in `PullModelStrategy.ALWAYS`): Always pull the model, even if it's already available. Useful to ensure you're using the latest version of the model. +* `when_missing` (defined in `PullModelStrategy.WHEN_MISSING`): Only pull the model if it's not already available. This may result in using an older version of the model. +* `never` (defined in `PullModelStrategy.NEVER`): Never pull the model automatically. -CAUTION: Due to the unexpected delays while downloading models, this feature is not recommended for production environments. Instead, consider to assess and pre-download the necessary models in advance. - -=== Pulling models at startup time +CAUTION: Due to potential delays while downloading models, automatic pulling is not recommended for production environments. Instead, consider assessing and pre-downloading the necessary models in advance. All models defined via configuration properties and default options can be automatically pulled at startup time. -You can configure strategy, timeout, and max number of retries via configuration properties. +You can configure the pull strategy, timeout, and maximum number of retries using configuration properties: [source,yaml] ---- @@ -190,9 +190,9 @@ spring: max-retries: 1 ---- -CAUTION: The application will not complete its initialization until all the models become available in Ollama. Depending on the model size and the speed of the Internet connection, your application might be slow at starting up. +CAUTION: The application will not complete its initialization until all specified models are available in Ollama. Depending on the model size and internet connection speed, this may significantly slow down your application's startup time. -You can also initialize additional models at startup time, useful for those models used dynamically at runtime. +You can initialize additional models at startup, which is useful for models used dynamically at runtime: [source,yaml] ---- @@ -207,7 +207,7 @@ spring: - nomic-embed-text ---- -If you want to apply the pulling strategy only to other types of models, you can exclude the embedding models from the initialization task. +If you want to apply the pulling strategy only to specific types of models, you can exclude embedding models from the initialization task: [source,yaml] ---- @@ -220,23 +220,7 @@ spring: include: false ---- -=== Pulling models at runtime - -To enable auto-pulling of models at runtime, you can configure the `pullModelStrategy` option in your `OllamaOptions`: - -[source,java] ----- -EmbeddingResponse embeddingResponse = embeddingModel - .call(new EmbeddingRequest(List.of("Hello World", "Something else"), - OllamaOptions.builder() - .withModel("all-minilm") - .withPullModelStrategy(PullModelStrategy.ALWAYS) - .build())); ----- - -You can also configure this option using the following property: `spring.ai.ollama.embedding.options.pull-model-strategy=always`. - -CAUTION: The time to process an incoming request might incur unexpected delays, waiting for the needed model to become available in Ollama. Depending on the model size and the speed of the Internet connection, your application might be slow at processing requests. You might want to initialize these models at startup time instead, using the `spring.ai.ollama.init.embedding.additional-models` property. +This configuration will apply the pulling strategy to all models except embedding models. == Sample Controller @@ -266,8 +250,12 @@ public class EmbeddingController { == Manual Configuration If you are not using Spring Boot, you can manually configure the `OllamaEmbeddingModel`. -For this add the spring-ai-ollama dependency to your project’s Maven pom.xml file: +For this add the spring-ai-ollama dependency to your project’s Maven pom.xml or Gradle `build.gradle` build files: +[tabs] +====== +Maven:: ++ [source,xml] ---- @@ -276,14 +264,15 @@ For this add the spring-ai-ollama dependency to your project’s Maven pom.xml f ---- -or to your Gradle `build.gradle` build file. - +Gradle:: ++ [source,groovy] ---- dependencies { implementation 'org.springframework.ai:spring-ai-ollama' } ---- +====== TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file. diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java index d4b621ee7..fb9db0f36 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/ollama/OllamaImage.java @@ -17,6 +17,6 @@ package org.springframework.ai.autoconfigure.ollama; public class OllamaImage { - public static final String IMAGE = "ollama/ollama:0.3.13"; + public static final String IMAGE = "ollama/ollama:0.3.14"; } diff --git a/spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaImage.java b/spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaImage.java index ed9a1aaab..59cf49403 100644 --- a/spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaImage.java +++ b/spring-ai-spring-boot-testcontainers/src/test/java/org/springframework/ai/testcontainers/service/connection/ollama/OllamaImage.java @@ -22,6 +22,6 @@ import org.testcontainers.utility.DockerImageName; */ public class OllamaImage { - public static final DockerImageName DEFAULT_IMAGE = DockerImageName.parse("ollama/ollama:0.3.13"); + public static final DockerImageName DEFAULT_IMAGE = DockerImageName.parse("ollama/ollama:0.3.14"); }