Add documentation for Image Generation

- Refine StabilityAI classes
- Add documentation
This commit is contained in:
Mark Pollack
2024-02-14 08:59:36 -05:00
parent 36278afdb3
commit 43dbaf4edd
17 changed files with 1007 additions and 323 deletions

View File

@@ -21,6 +21,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import org.springframework.ai.image.ImageOptions;
import org.springframework.ai.openai.api.OpenAiImageApi;
import java.util.Objects;
/**
* OpenAI Image API options. OpenAiImageOptions.java
*
@@ -44,6 +46,18 @@ public class OpenAiImageOptions implements ImageOptions {
@JsonProperty("model")
private String model = OpenAiImageApi.DEFAULT_IMAGE_MODEL;
/**
* The width of the generated images. Must be one of 256, 512, or 1024 for dall-e-2.
*/
@JsonProperty("size_width")
private Integer width;
/**
* The height of the generated images. Must be one of 256, 512, or 1024 for dall-e-2.
*/
@JsonProperty("size_height")
private Integer height;
/**
* The quality of the image that will be generated. hd creates images with finer
* details and greater consistency across the image. This param is only supported for
@@ -66,18 +80,6 @@ public class OpenAiImageOptions implements ImageOptions {
@JsonProperty("size")
private String size;
/**
* The width of the generated images. Must be one of 256, 512, or 1024 for dall-e-2.
*/
@JsonProperty("size_width")
private Integer width;
/**
* The height of the generated images. Must be one of 256, 512, or 1024 for dall-e-2.
*/
@JsonProperty("size_height")
private Integer height;
/**
* The style of the generated images. Must be one of vivid or natural. Vivid causes
* the model to lean towards generating hyper-real and dramatic images. Natural causes
@@ -235,4 +237,28 @@ public class OpenAiImageOptions implements ImageOptions {
return (this.width != null && this.height != null) ? this.width + "x" + this.height : null;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (!(o instanceof OpenAiImageOptions that))
return false;
return Objects.equals(n, that.n) && Objects.equals(model, that.model) && Objects.equals(width, that.width)
&& Objects.equals(height, that.height) && Objects.equals(quality, that.quality)
&& Objects.equals(responseFormat, that.responseFormat) && Objects.equals(size, that.size)
&& Objects.equals(style, that.style) && Objects.equals(user, that.user);
}
@Override
public int hashCode() {
return Objects.hash(n, model, width, height, quality, responseFormat, size, style, user);
}
@Override
public String toString() {
return "OpenAiImageOptions{" + "n=" + n + ", model='" + model + '\'' + ", width=" + width + ", height=" + height
+ ", quality='" + quality + '\'' + ", responseFormat='" + responseFormat + '\'' + ", size='" + size
+ '\'' + ", style='" + style + '\'' + ", user='" + user + '\'' + '}';
}
}

View File

@@ -21,8 +21,6 @@ import org.springframework.ai.image.*;
import org.springframework.ai.model.ModelOptionsUtils;
import org.springframework.ai.stabilityai.api.StabilityAiApi;
import org.springframework.ai.stabilityai.api.StabilityAiImageOptions;
import org.springframework.ai.stabilityai.api.StabilityAiImageOptionsBuilder;
import org.springframework.ai.stabilityai.api.StabilityAiImageOptionsImpl;
import org.springframework.util.Assert;
import java.util.List;
@@ -41,7 +39,7 @@ public class StabilityAiImageClient implements ImageClient {
private final StabilityAiApi stabilityAiApi;
public StabilityAiImageClient(StabilityAiApi stabilityAiApi) {
this(stabilityAiApi, StabilityAiImageOptionsBuilder.builder().build());
this(stabilityAiApi, StabilityAiImageOptions.builder().build());
}
public StabilityAiImageClient(StabilityAiApi stabilityAiApi, StabilityAiImageOptions options) {
@@ -71,7 +69,7 @@ public class StabilityAiImageClient implements ImageClient {
// options configured via Autoconfiguration.
// Runtime options overwrite StabilityAiImageClient options
StabilityAiImageOptions optionsToUse = ModelOptionsUtils.merge(runtimeOptions, this.options,
StabilityAiImageOptionsImpl.class);
StabilityAiImageOptions.class);
// Copy the org.springframework.ai.model derived ImagePrompt and ImageOptions data
// types to the data types used in StabilityAiApi
@@ -100,7 +98,7 @@ public class StabilityAiImageClient implements ImageClient {
.withCfgScale(optionsToUse.getCfgScale())
.withClipGuidancePreset(optionsToUse.getClipGuidancePreset())
.withSampler(optionsToUse.getSampler())
.withSamples(optionsToUse.getSamples())
.withSamples(optionsToUse.getN())
.withSeed(optionsToUse.getSeed())
.withSteps(optionsToUse.getSteps())
.withStylePreset(optionsToUse.getStylePreset())
@@ -118,7 +116,7 @@ public class StabilityAiImageClient implements ImageClient {
}
private StabilityAiImageOptions convertOptions(ImageOptions runtimeOptions) {
StabilityAiImageOptionsBuilder builder = StabilityAiImageOptionsBuilder.builder();
StabilityAiImageOptions.Builder builder = StabilityAiImageOptions.builder();
if (runtimeOptions == null) {
return builder.build();
}

View File

@@ -15,28 +15,476 @@
*/
package org.springframework.ai.stabilityai.api;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.springframework.ai.image.ImageOptions;
import org.springframework.ai.stabilityai.StyleEnum;
import java.util.Objects;
/**
* StabilityAiImageOptions is an interface that extends ImageOptions. It provides
* additional stability AI specific image options.
*/
public interface StabilityAiImageOptions extends ImageOptions {
@JsonInclude(JsonInclude.Include.NON_NULL)
public class StabilityAiImageOptions implements ImageOptions {
Float getCfgScale();
/**
* The number of images to be generated.
*
* Defaults to 1 if not explicitly set, indicating a single image will be generated.
*
* <p>
* This method specifies the total number of images to generate. It allows for
* controlling the volume of output from a single operation, facilitating batch
* generation of images based on the provided settings.
* </p>
*
* <p>
* Valid range of values: 1 to 10. This ensures that the request remains within a
* manageable scale and aligns with system capabilities or limitations.
* </p>
*
*
*/
@JsonProperty("samples")
private Integer n;
String getClipGuidancePreset();
/**
* The engine/model to use in Stability AI The model is passed in the URL as a path
* parameter
*
* The default value is stable-diffusion-v1-6
*/
private String model = StabilityAiApi.DEFAULT_IMAGE_MODEL;
String getSampler();
/**
* Retrieves the width of the image to be generated, in pixels.
* <p>
* Specifies the desired width for the output image. The value must be a multiple of
* 64 and at least 128 pixels. This parameter is adjusted to comply with the
* specifications of the selected generation engine, which may have unique
* requirements based on its version.
* </p>
*
* <p>
* Default value: 512.
* </p>
*
* <p>
* Engine-specific dimension validation:
* </p>
* <ul>
* <li>SDXL Beta: Width must be between 128 and 896 pixels, with only one dimension
* allowed to exceed 512.</li>
* <li>SDXL v0.9 and v1.0: Width must match one of the predefined dimension
* pairs.</li>
* <li>SD v1.6: Width must be between 320 and 1536 pixels.</li>
* </ul>
*
*/
@JsonProperty("width")
private Integer width;
Integer getSamples();
/**
* Retrieves the height of the image to be generated, in pixels.
* <p>
* Specifies the desired height for the output image. The value must be a multiple of
* 64 and at least 128 pixels. This setting is crucial for ensuring compatibility with
* the underlying generation engine, which may impose additional restrictions based on
* the engine version.
* </p>
*
* <p>
* Default value: 512.
* </p>
*
* <p>
* Engine-specific dimension validation:
* </p>
* <ul>
* <li>SDXL Beta: Height must be between 128 and 896 pixels, with only one dimension
* allowed to exceed 512.</li>
* <li>SDXL v0.9 and v1.0: Height must match one of the predefined dimension
* pairs.</li>
* <li>SD v1.6: Height must be between 320 and 1536 pixels.</li>
* </ul>
*
*/
@JsonProperty("height")
private Integer height;
Long getSeed();
/**
* The format in which the generated images are returned. It is sent as part of the
* accept header. Must be "application/json" or "image/png"
*/
private String responseFormat;
Integer getSteps();
/**
* The strictness level of the diffusion process adherence to the prompt text.
* <p>
* This field determines how closely the generated image will match the provided
* prompt. Higher values indicate that the image will adhere more closely to the
* prompt text, ensuring a closer match to the expected output.
* </p>
*
* <ul>
* <li>Range: 0 to 35</li>
* <li>Default value: 7</li>
* </ul>
*
*/
@JsonProperty("cfg_scale")
private Float cfgScale;
String getStylePreset();
/**
* The preset for clip guidance.
* <p>
* This field indicates the preset configuration for clip guidance, affecting the
* processing speed and characteristics. The choice of preset can influence the
* behavior of the guidance system, potentially impacting performance and output
* quality.
* </p>
*
* <p>
* Available presets are:
* <ul>
* <li>{@code FAST_BLUE}: An optimized preset for quicker processing with a focus on
* blue tones.</li>
* <li>{@code FAST_GREEN}: An optimized preset for quicker processing with a focus on
* green tones.</li>
* <li>{@code NONE}: No preset is applied, default processing.</li>
* <li>{@code SIMPLE}: A basic level of clip guidance for general use.</li>
* <li>{@code SLOW}: A slower processing preset for more detailed guidance.</li>
* <li>{@code SLOWER}: Further reduces the processing speed for enhanced detail in
* guidance.</li>
* <li>{@code SLOWEST}: The slowest processing speed, offering the highest level of
* detail in clip guidance.</li>
* </ul>
* </p>
*
* Defaults to {@code NONE} if no specific preset is configured.
*
*/
@JsonProperty("clip_guidance_preset")
private String clipGuidancePreset;
// extras json object...
/**
* The name of the sampler used for the diffusion process.
* <p>
* This field specifies the sampler algorithm to be used during the diffusion process.
* Selecting a specific sampler can influence the quality and characteristics of the
* generated output. If no sampler is explicitly selected, an appropriate sampler will
* be automatically chosen based on the context or other settings.
* </p>
*
* <p>
* Available samplers are:
* <ul>
* <li>{@code DDIM}: A deterministic diffusion inverse model for stable and
* predictable outputs.</li>
* <li>{@code DDPM}: Denoising diffusion probabilistic models for high-quality
* generation.</li>
* <li>{@code K_DPMPP_2M}: A specific configuration of DPM++ model with medium
* settings.</li>
* <li>{@code K_DPMPP_2S_ANCESTRAL}: An ancestral sampling variant of the DPM++ model
* with small settings.</li>
* <li>{@code K_DPM_2}: A variant of the DPM model designed for balanced
* performance.</li>
* <li>{@code K_DPM_2_ANCESTRAL}: An ancestral sampling variant of the DPM model.</li>
* <li>{@code K_EULER}: Utilizes the Euler method for diffusion, offering a different
* trade-off between speed and quality.</li>
* <li>{@code K_EULER_ANCESTRAL}: An ancestral version of the Euler method for nuanced
* sampling control.</li>
* <li>{@code K_HEUN}: Employs the Heun's method for a more accurate approximation in
* the diffusion process.</li>
* <li>{@code K_LMS}: Leverages the linear multistep method for potentially improved
* diffusion quality.</li>
* </ul>
* </p>
*
* An appropriate sampler is automatically selected if this value is omitted.
*
*/
@JsonProperty("sampler")
private String sampler;
/**
* The seed used for generating random noise.
* <p>
* This value serves as the seed for random noise generation, influencing the
* randomness and uniqueness of the output. A specific seed ensures reproducibility of
* results. Omitting this option or using 0 triggers the selection of a random seed.
* </p>
*
* <p>
* Valid range of values: 0 to 4294967295.
* </p>
*
* Default is 0, which indicates that a random seed will be used.
*/
@JsonProperty("seed")
private Long seed;
/**
* The number of diffusion steps to run.
* <p>
* Specifies the total number of steps in the diffusion process, affecting the detail
* and quality of the generated output. More steps can lead to higher quality but
* require more processing time.
* </p>
*
* <p>
* Valid range of values: 10 to 50.
* </p>
*
* Defaults to 30 if not explicitly set.
*/
@JsonProperty("steps")
private Integer steps;
/**
* The style preset intended to guide the image model towards a specific artistic
* style.
* <p>
* This string parameter allows for the selection of a predefined style preset,
* influencing the aesthetic characteristics of the generated image. The choice of
* preset can significantly impact the visual outcome, aligning it with particular
* artistic genres or techniques.
* </p>
*
* <p>
* Possible values include:
* </p>
* <ul>
* <li>{@code 3d-model}</li>
* <li>{@code analog-film}</li>
* <li>{@code anime}</li>
* <li>{@code cinematic}</li>
* <li>{@code comic-book}</li>
* <li>{@code digital-art}</li>
* <li>{@code enhance}</li>
* <li>{@code fantasy-art}</li>
* <li>{@code isometric}</li>
* <li>{@code line-art}</li>
* <li>{@code low-poly}</li>
* <li>{@code modeling-compound}</li>
* <li>{@code neon-punk}</li>
* <li>{@code origami}</li>
* <li>{@code photographic}</li>
* <li>{@code pixel-art}</li>
* <li>{@code tile-texture}</li>
* </ul>
* <p>
* Note: This list of style presets is subject to change.
* </p>
*
*/
@JsonProperty("style_preset")
private String stylePreset;
public static Builder builder() {
return new Builder();
}
public static class Builder {
private final StabilityAiImageOptions options;
private Builder() {
this.options = new StabilityAiImageOptions();
}
public Builder withN(Integer n) {
options.setN(n);
return this;
}
public Builder withModel(String model) {
options.setModel(model);
return this;
}
public Builder withWidth(Integer width) {
options.setWidth(width);
return this;
}
public Builder withHeight(Integer height) {
options.setHeight(height);
return this;
}
public Builder withResponseFormat(String responseFormat) {
options.setResponseFormat(responseFormat);
return this;
}
public Builder withCfgScale(Float cfgScale) {
options.setCfgScale(cfgScale);
return this;
}
public Builder withClipGuidancePreset(String clipGuidancePreset) {
options.setClipGuidancePreset(clipGuidancePreset);
return this;
}
public Builder withSampler(String sampler) {
options.setSampler(sampler);
return this;
}
public Builder withSeed(Long seed) {
options.setSeed(seed);
return this;
}
public Builder withSteps(Integer steps) {
options.setSteps(steps);
return this;
}
public Builder withSamples(Integer samples) {
options.setN(samples);
return this;
}
public Builder withStylePreset(String stylePreset) {
options.setStylePreset(stylePreset);
return this;
}
public Builder withStylePreset(StyleEnum styleEnum) {
options.setStylePreset(styleEnum.toString());
return this;
}
public StabilityAiImageOptions build() {
return options;
}
}
@Override
public Integer getN() {
return n;
}
public void setN(Integer n) {
this.n = n;
}
@Override
public String getModel() {
return model;
}
public void setModel(String model) {
this.model = model;
}
@Override
public Integer getWidth() {
return width;
}
public void setWidth(Integer width) {
this.width = width;
}
@Override
public Integer getHeight() {
return height;
}
public void setHeight(Integer height) {
this.height = height;
}
@Override
public String getResponseFormat() {
return responseFormat;
}
public void setResponseFormat(String responseFormat) {
this.responseFormat = responseFormat;
}
public Float getCfgScale() {
return cfgScale;
}
public void setCfgScale(Float cfgScale) {
this.cfgScale = cfgScale;
}
public String getClipGuidancePreset() {
return clipGuidancePreset;
}
public void setClipGuidancePreset(String clipGuidancePreset) {
this.clipGuidancePreset = clipGuidancePreset;
}
public String getSampler() {
return sampler;
}
public void setSampler(String sampler) {
this.sampler = sampler;
}
public Long getSeed() {
return seed;
}
public void setSeed(Long seed) {
this.seed = seed;
}
public Integer getSteps() {
return steps;
}
public void setSteps(Integer steps) {
this.steps = steps;
}
public String getStylePreset() {
return stylePreset;
}
public void setStylePreset(String stylePreset) {
this.stylePreset = stylePreset;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (!(o instanceof StabilityAiImageOptions that))
return false;
return Objects.equals(n, that.n) && Objects.equals(model, that.model) && Objects.equals(width, that.width)
&& Objects.equals(height, that.height) && Objects.equals(responseFormat, that.responseFormat)
&& Objects.equals(cfgScale, that.cfgScale)
&& Objects.equals(clipGuidancePreset, that.clipGuidancePreset) && Objects.equals(sampler, that.sampler)
&& Objects.equals(seed, that.seed) && Objects.equals(steps, that.steps)
&& Objects.equals(stylePreset, that.stylePreset);
}
@Override
public int hashCode() {
return Objects.hash(n, model, width, height, responseFormat, cfgScale, clipGuidancePreset, sampler, seed, steps,
stylePreset);
}
@Override
public String toString() {
return "StabilityAiImageOptions{" + "n=" + n + ", model='" + model + '\'' + ", width=" + width + ", height="
+ height + ", responseFormat='" + responseFormat + '\'' + ", cfgScale=" + cfgScale
+ ", clipGuidancePreset='" + clipGuidancePreset + '\'' + ", sampler='" + sampler + '\'' + ", seed="
+ seed + ", steps=" + steps + ", stylePreset='" + stylePreset + '\'' + '}';
}
}

View File

@@ -1,106 +0,0 @@
/*
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.stabilityai.api;
import org.springframework.ai.stabilityai.StyleEnum;
/**
* The StabilityAiImageOptionsBuilder class provides a convenient way to construct an
* instance of StabilityAiImageOptions. by allowing you to chain multiple method calls to
* set the desired options.
*/
public class StabilityAiImageOptionsBuilder {
private StabilityAiImageOptionsImpl options;
private StabilityAiImageOptionsBuilder() {
options = new StabilityAiImageOptionsImpl();
}
public static StabilityAiImageOptionsBuilder builder() {
return new StabilityAiImageOptionsBuilder();
}
public StabilityAiImageOptionsBuilder withN(Integer n) {
options.setN(n);
return this;
}
public StabilityAiImageOptionsBuilder withModel(String model) {
options.setModel(model);
return this;
}
public StabilityAiImageOptionsBuilder withWidth(Integer width) {
options.setWidth(width);
return this;
}
public StabilityAiImageOptionsBuilder withHeight(Integer height) {
options.setHeight(height);
return this;
}
public StabilityAiImageOptionsBuilder withResponseFormat(String responseFormat) {
options.setResponseFormat(responseFormat);
return this;
}
public StabilityAiImageOptionsBuilder withCfgScale(Float cfgScale) {
options.setCfgScale(cfgScale);
return this;
}
public StabilityAiImageOptionsBuilder withClipGuidancePreset(String clipGuidancePreset) {
options.setClipGuidancePreset(clipGuidancePreset);
return this;
}
public StabilityAiImageOptionsBuilder withSampler(String sampler) {
options.setSampler(sampler);
return this;
}
public StabilityAiImageOptionsBuilder withSeed(Long seed) {
options.setSeed(seed);
return this;
}
public StabilityAiImageOptionsBuilder withSteps(Integer steps) {
options.setSteps(steps);
return this;
}
public StabilityAiImageOptionsBuilder withSamples(Integer samples) {
options.setSamples(samples);
return this;
}
public StabilityAiImageOptionsBuilder withStylePreset(String stylePreset) {
options.setStylePreset(stylePreset);
return this;
}
public StabilityAiImageOptionsBuilder withStylePreset(StyleEnum styleEnum) {
options.setStylePreset(styleEnum.toString());
return this;
}
public StabilityAiImageOptions build() {
return options;
}
}

View File

@@ -1,155 +0,0 @@
/*
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.stabilityai.api;
public class StabilityAiImageOptionsImpl implements StabilityAiImageOptions {
private Integer n;
private String model;
private Integer width;
private Integer height;
private String responseFormat;
private Float cfgScale;
private String clipGuidancePreset;
private String sampler;
private Integer samples;
private Long seed;
private Integer steps;
private String stylePreset;
public StabilityAiImageOptionsImpl() {
}
@Override
public Integer getN() {
return this.n;
}
public void setN(Integer n) {
this.n = n;
}
@Override
public String getModel() {
return this.model;
}
public void setModel(String model) {
this.model = model;
}
@Override
public Integer getWidth() {
return this.width;
}
public void setWidth(Integer width) {
this.width = width;
}
@Override
public Integer getHeight() {
return this.height;
}
public void setHeight(Integer height) {
this.height = height;
}
@Override
public String getResponseFormat() {
return this.responseFormat;
}
public void setResponseFormat(String responseFormat) {
this.responseFormat = responseFormat;
}
@Override
public Float getCfgScale() {
return this.cfgScale;
}
public void setCfgScale(Float cfgScale) {
this.cfgScale = cfgScale;
}
@Override
public String getClipGuidancePreset() {
return this.clipGuidancePreset;
}
public void setClipGuidancePreset(String clipGuidancePreset) {
this.clipGuidancePreset = clipGuidancePreset;
}
@Override
public String getSampler() {
return this.sampler;
}
public void setSampler(String sampler) {
this.sampler = sampler;
}
@Override
public Integer getSamples() {
return this.samples;
}
public void setSamples(Integer samples) {
this.samples = samples;
}
@Override
public Long getSeed() {
return this.seed;
}
public void setSeed(Long seed) {
this.seed = seed;
}
@Override
public Integer getSteps() {
return this.steps;
}
public void setSteps(Integer steps) {
this.steps = steps;
}
@Override
public String getStylePreset() {
return this.stylePreset;
}
public void setStylePreset(String stylePreset) {
this.stylePreset = stylePreset;
}
}

View File

@@ -18,14 +18,11 @@ package org.springframework.ai.stabilityai;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.ai.image.*;
import org.springframework.ai.stabilityai.api.StabilityAiApi;
import org.springframework.ai.stabilityai.api.StabilityAiImageOptions;
import org.springframework.ai.stabilityai.api.StabilityAiImageOptionsBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Base64;
@@ -42,12 +39,13 @@ public class StabilityAiImageClientIT {
@Test
void imageAsBase64Test() throws IOException {
StabilityAiImageOptions imageOptions = StabilityAiImageOptionsBuilder.builder()
StabilityAiImageOptions imageOptions = StabilityAiImageOptions.builder()
.withStylePreset(StyleEnum.PHOTOGRAPHIC)
.build();
var instructions = """
A light cream colored mini golden doodle with a sign that contains the message "I'm on my way to BARCADE!".""";
A light cream colored mini golden doodle.
""";
ImagePrompt imagePrompt = new ImagePrompt(instructions, imageOptions);
@@ -58,7 +56,7 @@ public class StabilityAiImageClientIT {
assertThat(image.getB64Json()).isNotEmpty();
// writeFile(image);
writeFile(image);
}
private static void writeFile(Image image) throws IOException {

View File

@@ -18,6 +18,10 @@ package org.springframework.ai.image;
import org.springframework.ai.model.ModelOptions;
/**
* ImageOptions represent the common options, portable across different image generation
* models.
*/
public interface ImageOptions extends ModelOptions {
Integer getN();

View File

@@ -24,6 +24,9 @@
**** xref:api/clients/bedrock/bedrock-titan.adoc[]
*** xref:api/clients/huggingface.adoc[]
*** xref:api/clients/vertexai-chat.adoc[]
** xref:api/imageclient.adoc[]
*** xref:api/clients/image/openai-image.adoc[]
*** xref:api/clients/image/stabilityai-image.adoc[]
** xref:api/prompt.adoc[]
** xref:api/output-parser.adoc[]
** xref:api/vectordbs.adoc[]

View File

@@ -0,0 +1,95 @@
= OpenAI Image Generation
Spring AI supports ChatGPT's DALL-E, the Image generation model from OpenAI.
== Prerequisites
You will need to create an API key with OpenAI to access ChatGPT models.
Create an account at https://platform.openai.com/signup[OpenAI signup page] and generate the token on the https://platform.openai.com/account/api-keys[API Keys page].
The Spring AI project defines a configuration property named `spring.ai.openai.api-key` that you should set to the value of the `API Key` obtained from openai.com.
Exporting an environment variable is one way to set that configuration property:
[source,shell]
----
export SPRING_AI_OPENAI_API_KEY=<INSERT KEY HERE>
----
== Auto-configuration
Spring AI provides Spring Boot auto-configuration for the OpenAI Image Generation Client.
To enable it add the following dependency to your project's Maven `pom.xml` file:
[source, xml]
----
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-openai-spring-boot-starter</artifactId>
<version>0.8.0-SNAPSHOT</version>
</dependency>
----
or to your Gradle `build.gradle` build file.
[source,groovy]
----
dependencies {
implementation 'org.springframework.ai:spring-ai-openai-spring-boot-starter:0.8.0-SNAPSHOT'
}
----
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
=== Image Generation Properties
The prefix `spring.ai.openai` is used as the property prefix that lets you connect to OpenAI.
[cols="3,5,1"]
|====
| Property | Description | Default
| spring.ai.openai.base-url | The URL to connect to | https://api.openai.com
| spring.ai.openai.api-key | The API Key | -
|====
The prefix `spring.ai.openai.image` is the property prefix that lets you configure the `ImageClient` implementation for OpenAI.
[cols="3,5,1"]
|====
| Property | Description | Default
| spring.ai.openai.chat.base-url | Optional overrides the spring.ai.openai.base-url to provide chat specific url | -
| spring.ai.openai.chat.api-key | Optional overrides the spring.ai.openai.api-key to provide chat specific api-key | -
| spring.ai.openai.image.options.n | The number of images to generate. Must be between 1 and 10. For dall-e-3, only n=1 is supported. | -
| spring.ai.openai.image.options.model | The model to use for image generation. | OpenAiImageApi.DEFAULT_IMAGE_MODEL
| spring.ai.openai.image.options.quality | The quality of the image that will be generated. HD creates images with finer details and greater consistency across the image. This parameter is only supported for dall-e-3. | -
| spring.ai.openai.image.options.response_format | The format in which the generated images are returned. Must be one of URL or b64_json. | -
| `spring.ai.openai.image.options.size` | The size of the generated images. Must be one of 256x256, 512x512, or 1024x1024 for dall-e-2. Must be one of 1024x1024, 1792x1024, or 1024x1792 for dall-e-3 models. | -
| `spring.ai.openai.image.options.size_width` | The width of the generated images. Must be one of 256, 512, or 1024 for dall-e-2. | -
| `spring.ai.openai.image.options.size_height`| The height of the generated images. Must be one of 256, 512, or 1024 for dall-e-2. | -
| `spring.ai.openai.image.options.style` | The style of the generated images. Must be one of vivid or natural. Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images. This parameter is only supported for dall-e-3. | -
| `spring.ai.openai.image.options.user` | A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. | -
|====
=== Image Options [[image-options]]
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiImageOptions.java[OpenAiImageOptions.java] provides model configurations, such as the model to use, the quality, the size, etc.
On start-up, the default options can be configured with the `OpenAiImageClient(OpenAiImageApi openAiImageApi)` constructor and the `withDefaultOptions(OpenAiImageOptions defaultOptions)` method. Alternatively, use the `spring.ai.openai.image.options.*` properties described previously.
At run-time you can override the default options by adding new, request specific, options to the `ImagePrompt` call.
For example to override the OpenAI specific options such as quality and the number of images to create, use the following code example:
[source,java]
----
ImageResponse response = openaiImageClient.call(
new ImagePrompt("A light cream colored mini golden doodle",
OpenAiImageOptions.builder()
.withQuality("hd")
.withN(4)
.withHeight(1024)
.withWidth(1024).build())
);
----
TIP: In addition to the model specific https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiImageOptions.java[OpenAiImageOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageOptions.java[ImageOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageOptionsBuilder.java[ImageOptionsBuilder#builder()].

View File

@@ -0,0 +1,99 @@
= Stability AI Image Generation
Spring AI supports Stability AI's https://platform.stability.ai/docs/api-reference#tag/v1generation[text to image generation model].
== Prerequisites
You will need to create an API key with Stability AI to access their AI models, follow their https://platform.stability.ai/docs/getting-started/authentication[Getting Started documentation].
The Spring AI project defines a configuration property named `spring.ai.stabilityai.api-key` that you should set to the value of the `API Key` obtained from Stability AI.
Exporting an environment variable in one way to set that configuration property.
[source,shell]
----
export SPRING_AI_STABILITYAI_API_KEY=<INSERT KEY HERE>
----
== Auto-configuration
Spring AI provides Spring Boot auto-configuration for the Stability AI Image Generation Client.
To enable it add the following dependency to your project's Maven `pom.xml` file:
[source, xml]
----
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-stability-ai-spring-boot-starter</artifactId>
<version>0.8.0-SNAPSHOT</version>
</dependency>
----
or to your Gradle `build.gradle` build file.
[source,groovy]
----
dependencies {
implementation 'org.springframework.ai:spring-ai-stability-ai-spring-boot-starter:0.8.0-SNAPSHOT'
}
----
TIP: Refer to the xref:getting-started.adoc#_dependency_management[Dependency Management] section to add Milestone and/or Snapshot Repositories to your build file.
=== Image Generation Properties
The prefix `spring.ai.stabilityai` is used as the property prefix that lets you connect to Stability AI.
[cols="3,5,1"]
|====
| Property | Description | Default
| spring.ai.stabilityai.base-url | The URL to connect to | https://api.stability.ai/v1
| spring.ai.stabilityai.api-key | The API Key | -
|====
The prefix `spring.ai.stabilityai.image` is the property prefix that lets you configure the `ImageClient` implementation for Stability AI.
[cols="2,5,1"]
|====
| Property | Description | Default
| `spring.ai.stabilityai.image.base-url` | Optional overrides the spring.ai.openai.base-url to provide a specific url | `https://api.stability.ai/v1`
| `spring.ai.stabilityai.image.api-key` | Optional overrides the spring.ai.openai.api-key to provide a specific api-key | -
| `spring.ai.stabilityai.image.option.n` | The number of images to be generated. Must be between 1 and 10. | 1
| `spring.ai.stabilityai.image.option.model` | The engine/model to use in Stability AI. The model is passed in the URL as a path parameter. | `stable-diffusion-v1-6`
| `spring.ai.stabilityai.image.option.width` | Width of the image to generate, in pixels, in an increment divisible by 64. Engine-specific dimension validation applies. | 512
| `spring.ai.stabilityai.image.option.height` | Height of the image to generate, in pixels, in an increment divisible by 64. Engine-specific dimension validation applies.| 512
| `spring.ai.stabilityai.image.option.responseFormat` | The format in which the generated images are returned. Must be "application/json" or "image/png". | -
| `spring.ai.stabilityai.image.option.cfg_scale` | The strictness level of the diffusion process adherence to the prompt text. Range: 0 to 35. | 7
| `spring.ai.stabilityai.image.option.clip_guidance_preset` | Pass in a style preset to guide the image model towards a particular style. This list of style presets is subject to change. | `NONE`
| `spring.ai.stabilityai.image.option.sampler` | Which sampler to use for the diffusion process. If this value is omitted, an appropriate sampler will be automatically selected. | -
| `spring.ai.stabilityai.image.option.seed` | Random noise seed (omit this option or use 0 for a random seed). Valid range: 0 to 4294967295. | 0
| `spring.ai.stabilityai.image.option.steps` | Number of diffusion steps to run. Valid range: 10 to 50. | 30
| `spring.ai.stabilityai.image.option.style_preset` | Pass in a style preset to guide the image model towards a particular style. This list of style presets is subject to change. | -
|====
=== Image Options [[image-options]]
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-stabilityai/src/main/java/org/springframework/ai/stabilityai/api/StabilityAiImageOptions.java[StabilityAiImageOptions.java] provides model configurations, such as the model to use, the style, the size, etc.
On start-up, the default options can be configured with the `StabilityAiImageClient(StabilityAiApi stabilityAiApi, StabilityAiImageOptions options)` constructor. Alternatively, use the `spring.ai.openai.image.options.*` properties described previously.
At run-time you can override the default options by adding new, request specific, options to the `ImagePrompt` call.
For example to override the Stability AI specific options such as quality and the number of images to create, use the following code example:
[source,java]
----
ImageResponse response = openaiImageClient.call(
new ImagePrompt("A light cream colored mini golden doodle",
StabilityAiImageOptions.builder()
.withStylePreset("cinematic")
.withN(4)
.withHeight(1024)
.withWidth(1024).build())
);
----
TIP: In addition to the model specific https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-stabilityai/src/main/java/org/springframework/ai/stabilityai/api/StabilityAiImageOptions.java[StabilityAiImageOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageOptions.java[ImageOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageOptionsBuilder.java[ImageOptionsBuilder#builder()].

View File

@@ -0,0 +1,171 @@
[[ImageClient]]
= Image Generation API
The `Spring Image Generation API` is designed to be a simple and portable interface for interacting with various xref:concepts.adoc#_models[AI Models] specialized in image generation, allowing developers to switch between different image-related models with minimal code changes.
This design aligns with Spring's philosophy of modularity and interchangeability, ensuring developers can quickly adapt their applications to different AI capabilities related to image processing.
Additionally, with the support of companion classes like `ImagePrompt` for input encapsulation and `ImageResponse` for output handling, the Image Generation API unifies the communication with AI Models dedicated to image generation.
It manages the complexity of request preparation and response parsing, offering a direct and simplified API interaction for image-generation functionalities.
The Spring Image Generation API is built on top of the Spring AI `Generic Model API`, providing image-specific abstractions and implementations.
== API Overview
This section provides a guide to the Spring Image Generation API interface and associated classes.
== Image Client
Here is the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageClient.java[ImageClient] interface definition:
[source,java]
----
@FunctionalInterface
public interface ImageClient extends ModelClient<ImagePrompt, ImageResponse> {
ImageResponse call(ImagePrompt request);
}
----
=== ImagePrompt
The https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImagePrompt.java[ImagePrompt] is a `ModelRequest` that encapsulates a list of https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageMessage.java[ImageMessage] objects and optional model request options.
The following listing shows a truncated version of the `ImagePrompt` class, excluding constructors and other utility methods:
[source,java]
----
public class ImagePrompt implements ModelRequest<List<ImageMessage>> {
private final List<ImageMessage> messages;
private ImageOptions imageModelOptions;
@Override
public List<ImageMessage> getInstructions() {...}
@Override
public ImageOptions getOptions() {...}
// constructors and utility methods omitted
}
----
==== ImageMessage
The `ImageMessage` class encapsulates the text to use and the weight that the text should have in influencing the generated image. For models that support weights, they can be positive or negative.
[source,java]
----
public class ImageMessage {
private String text;
private Float weight;
public String getText() {...}
public Float getWeight() {...}
// constructors and utility methods omitted
----
==== ImageOptions
Represents the options that can be passed to the Image generation model. The `ImageOptions` class extends the `ModelOptions` interface and is used to define few portable options that can be passed to the AI model.
The `ImageOptions` class is defined as follows:
[source,java]
----
public interface ImageOptions extends ModelOptions {
Integer getN();
String getModel();
Integer getWidth();
Integer getHeight();
String getResponseFormat(); // openai - url or base64 : stability ai byte[] or base64
}
----
Additionally, every model specific ImageClient implementation can have its own options that can be passed to the AI model. For example, the OpenAI Image Generation model has its own options like `quality`, `style`, etc.
This is a powerful feature that allows developers to use model specific options when starting the application and then override them with at runtime using the `ImagePrompt`.
=== ImageResponse
The structure of the `ChatResponse` class is as follows:
[source,java]
----
public class ImageResponse implements ModelResponse<ImageGeneration> {
private final ImageResponseMetadata imageResponseMetadata;
private final List<ImageGeneration> imageGenerations;
@Override
public ImageGeneration getResult() {
// get the first result
}
@Override
public List<ImageGeneration> getResults() {...}
@Override
public ImageResponseMetadata getMetadata() {...}
// other methods omitted
}
----
The https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageResponse.java[ImageResponse] class holds the AI Model's output, with each `ImageGeneration` instance containing one of potentially multiple outputs resulting from a single prompt.
The `ImageResponse` class also carries a `ImageResponseMetadata` metadata about the AI Model's response.
=== ImageGeneration
Finally, the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageGeneration.java[ImageGeneration] class extends from the `ModelResult` to represent the output response and related metadata about this result:
[source,java]
----
public class ImageGeneration implements ModelResult<Image> {
private ImageGenerationMetadata imageGenerationMetadata;
private Image image;
@Override
public Image getOutput() {...}
@Override
public ImageGenerationMetadata getMetadata() {...}
// other methods omitted
}
----
== Available Implementations
`ImageClient` implementations are provided for the following Model providers:
* xref:api/clients/image/openai-image.adoc[OpenAI Image Generation]
* xref:api/clients/image/stabilityai-image.adoc[StabilityAI Image Generation]
== API Docs
You can find the Javadoc https://docs.spring.io/spring-ai/docs/current-SNAPSHOT/[here].
== Feedback and Contributions
The project's https://github.com/spring-projects/spring-ai/discussions[GitHub discussions] is a great place to send feedback.

View File

@@ -3,6 +3,7 @@
This section offers jumping off points for how to get started using Spring AI.
[#_dependency_management]
== Dependency Management
The Spring AI project provides artifacts in the Spring Milestone and Snapshot repositories.

View File

@@ -0,0 +1,33 @@
/*
* Copyright 2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.autoconfigure.stabilityai;
import org.springframework.ai.stabilityai.api.StabilityAiApi;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(StabilityAiConnectionProperties.CONFIG_PREFIX)
public class StabilityAiConnectionProperties extends StabilityAiParentProperties {
public static final String CONFIG_PREFIX = "spring.ai.stabilityai";
public static final String DEFAULT_BASE_URL = StabilityAiApi.DEFAULT_BASE_URL;
public StabilityAiConnectionProperties() {
super.setBaseUrl(DEFAULT_BASE_URL);
}
}

View File

@@ -33,23 +33,23 @@ import org.springframework.web.client.RestClient;
*/
@AutoConfiguration(after = RestClientAutoConfiguration.class)
@ConditionalOnClass(StabilityAiApi.class)
@EnableConfigurationProperties({ StabilityAiProperties.class })
@EnableConfigurationProperties({ StabilityAiImageProperties.class })
@ImportRuntimeHints(NativeHints.class)
public class StabilityAiImageAutoConfiguration {
@Bean
@ConditionalOnMissingBean
public StabilityAiApi stabilityAiApi(StabilityAiProperties stabilityAiProperties,
public StabilityAiApi stabilityAiApi(StabilityAiImageProperties stabilityAiImageProperties,
RestClient.Builder restClientBuilder) {
return new StabilityAiApi(stabilityAiProperties.getApiKey(), stabilityAiProperties.getBaseUrl(),
stabilityAiProperties.getOptions().getModel(), restClientBuilder);
return new StabilityAiApi(stabilityAiImageProperties.getApiKey(), stabilityAiImageProperties.getBaseUrl(),
stabilityAiImageProperties.getOptions().getModel(), restClientBuilder);
}
@Bean
@ConditionalOnMissingBean
public StabilityAiImageClient stabilityAiImageClient(StabilityAiApi stabilityAiApi,
StabilityAiProperties stabilityAiProperties) {
return new StabilityAiImageClient(stabilityAiApi, stabilityAiProperties.getOptions());
StabilityAiImageProperties stabilityAiImageProperties) {
return new StabilityAiImageClient(stabilityAiApi, stabilityAiImageProperties.getOptions());
}
}

View File

@@ -24,34 +24,14 @@ import org.springframework.boot.context.properties.NestedConfigurationProperty;
* @author Mark Pollack
* @since 0.8.0
*/
@ConfigurationProperties(StabilityAiProperties.CONFIG_PREFIX)
public class StabilityAiProperties {
@ConfigurationProperties(StabilityAiImageProperties.CONFIG_PREFIX)
public class StabilityAiImageProperties extends StabilityAiParentProperties {
public static final String CONFIG_PREFIX = "spring.ai.stabilityai";
private String apiKey;
private String baseUrl = StabilityAiApi.DEFAULT_BASE_URL;
public static final String CONFIG_PREFIX = "spring.ai.stabilityai.image";
@NestedConfigurationProperty
private StabilityAiImageOptions options;
public String getApiKey() {
return this.apiKey;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
public String getBaseUrl() {
return this.baseUrl;
}
public void setBaseUrl(String baseUrl) {
this.baseUrl = baseUrl;
}
public StabilityAiImageOptions getOptions() {
return this.options;
}

View File

@@ -0,0 +1,47 @@
/*
* Copyright 2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.autoconfigure.stabilityai;
/**
* Internal parent properties for the StabilityAI properties.
*
* @author Mark Pollack
* @since 0.8.0
*/
class StabilityAiParentProperties {
private String apiKey;
private String baseUrl;
public String getApiKey() {
return apiKey;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
public String getBaseUrl() {
return baseUrl;
}
public void setBaseUrl(String baseUrl) {
this.baseUrl = baseUrl;
}
}

View File

@@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai</artifactId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spring-ai-stability-ai-spring-boot-starter</artifactId>
<packaging>jar</packaging>
<name>Spring AI Starter - Stability AI</name>
<description>Spring AI Stability Auto Configuration</description>
<url>https://github.com/spring-projects/spring-ai</url>
<scm>
<url>https://github.com/spring-projects/spring-ai</url>
<connection>git://github.com/spring-projects/spring-ai.git</connection>
<developerConnection>git@github.com:spring-projects/spring-ai.git</developerConnection>
</scm>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-spring-boot-autoconfigure</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-stability-ai</artifactId>
<version>${project.parent.version}</version>
</dependency>
</dependencies>
</project>