Add the support of Azure OpenAI image generation
* Add docs
This commit is contained in:
Benoit Moussaud
committed by
Mark Pollack
Mark Pollack
parent
574138a26a
commit
a30cc9d84d
@@ -0,0 +1,190 @@
|
||||
package org.springframework.ai.azure.openai;
|
||||
|
||||
import com.azure.ai.openai.OpenAIClient;
|
||||
import com.azure.ai.openai.models.ImageGenerationOptions;
|
||||
import com.azure.ai.openai.models.ImageGenerationQuality;
|
||||
import com.azure.ai.openai.models.ImageGenerationResponseFormat;
|
||||
import com.azure.ai.openai.models.ImageGenerationStyle;
|
||||
import com.azure.ai.openai.models.ImageSize;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.azure.openai.metadata.AzureOpenAiImageGenerationMetadata;
|
||||
import org.springframework.ai.azure.openai.metadata.AzureOpenAiImageResponseMetadata;
|
||||
import org.springframework.ai.image.Image;
|
||||
import org.springframework.ai.image.ImageGeneration;
|
||||
import org.springframework.ai.image.ImageModel;
|
||||
import org.springframework.ai.image.ImagePrompt;
|
||||
import org.springframework.ai.image.ImageResponse;
|
||||
import org.springframework.ai.image.ImageResponseMetadata;
|
||||
import org.springframework.ai.model.ModelOptionsUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
/**
|
||||
* {@link ImageModel} implementation for {@literal Microsoft Azure AI} backed by
|
||||
* {@link OpenAIClient}.
|
||||
*
|
||||
* @author Benoit Moussaud
|
||||
* @see ImageModel
|
||||
* @see com.azure.ai.openai.OpenAIClient
|
||||
* @since 1.0.0 M1
|
||||
*/
|
||||
public class AzureOpenAiImageModel implements ImageModel {
|
||||
|
||||
private static final String DEFAULT_DEPLOYMENT_NAME = AzureOpenAiImageOptions.DEFAULT_IMAGE_MODEL;
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(getClass());
|
||||
|
||||
@Autowired
|
||||
private final OpenAIClient openAIClient;
|
||||
|
||||
private final AzureOpenAiImageOptions defaultOptions;
|
||||
|
||||
public AzureOpenAiImageModel(OpenAIClient openAIClient) {
|
||||
this(openAIClient, AzureOpenAiImageOptions.builder().withDeploymentName(DEFAULT_DEPLOYMENT_NAME).build());
|
||||
}
|
||||
|
||||
public AzureOpenAiImageModel(OpenAIClient microsoftOpenAiClient, AzureOpenAiImageOptions options) {
|
||||
Assert.notNull(microsoftOpenAiClient, "com.azure.ai.openai.OpenAIClient must not be null");
|
||||
Assert.notNull(options, "AzureOpenAiChatOptions must not be null");
|
||||
this.openAIClient = microsoftOpenAiClient;
|
||||
this.defaultOptions = options;
|
||||
}
|
||||
|
||||
public AzureOpenAiImageOptions getDefaultOptions() {
|
||||
return defaultOptions;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImageResponse call(ImagePrompt imagePrompt) {
|
||||
ImageGenerationOptions imageGenerationOptions = toOpenAiImageOptions(imagePrompt);
|
||||
String deploymentOrModelName = getDeploymentName(imagePrompt);
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Azure ImageGenerationOptions call {} with the following options : {} ", deploymentOrModelName,
|
||||
toPrettyJson(imageGenerationOptions));
|
||||
}
|
||||
|
||||
var images = openAIClient.getImageGenerations(deploymentOrModelName, imageGenerationOptions);
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Azure ImageGenerations: {}", toPrettyJson(images));
|
||||
}
|
||||
|
||||
List<ImageGeneration> imageGenerations = images.getData().stream().map(entry -> {
|
||||
var image = new Image(entry.getUrl(), entry.getBase64Data());
|
||||
var metadata = new AzureOpenAiImageGenerationMetadata(entry.getRevisedPrompt());
|
||||
return new ImageGeneration(image, metadata);
|
||||
}).toList();
|
||||
|
||||
ImageResponseMetadata openAiImageResponseMetadata = AzureOpenAiImageResponseMetadata.from(images);
|
||||
return new ImageResponse(imageGenerations, openAiImageResponseMetadata);
|
||||
}
|
||||
|
||||
private String toPrettyJson(Object object) {
|
||||
ObjectMapper objectMapper = new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
|
||||
.registerModule(new JavaTimeModule());
|
||||
try {
|
||||
return objectMapper.writeValueAsString(object);
|
||||
}
|
||||
catch (JsonProcessingException e) {
|
||||
return "JsonProcessingException:" + e + " [" + object.toString() + "]";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the deployment-name if provided or use the model name.
|
||||
* @param prompt the image prompt
|
||||
* @return Return the deployment-name if provided or use the model name.
|
||||
*/
|
||||
private String getDeploymentName(ImagePrompt prompt) {
|
||||
var runtimeImageOptions = prompt.getOptions();
|
||||
|
||||
if (this.defaultOptions != null) {
|
||||
// Merge options fixed in beta7
|
||||
// https://github.com/Azure/azure-sdk-for-java/issues/38183
|
||||
runtimeImageOptions = ModelOptionsUtils.merge(runtimeImageOptions, this.defaultOptions,
|
||||
AzureOpenAiImageOptions.class);
|
||||
}
|
||||
|
||||
if (runtimeImageOptions != null) {
|
||||
if (runtimeImageOptions instanceof AzureOpenAiImageOptions runtimeAzureOpenAiImageOptions) {
|
||||
if (runtimeAzureOpenAiImageOptions.getDeploymentName() != null) {
|
||||
return runtimeAzureOpenAiImageOptions.getDeploymentName();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// By default the one provided in the image prompt
|
||||
return prompt.getOptions().getModel();
|
||||
|
||||
}
|
||||
|
||||
private ImageGenerationOptions toOpenAiImageOptions(ImagePrompt prompt) {
|
||||
|
||||
if (prompt.getInstructions().size() > 1) {
|
||||
throw new RuntimeException(format("implementation support 1 image instruction only, found %s",
|
||||
prompt.getInstructions().size()));
|
||||
}
|
||||
if (prompt.getInstructions().isEmpty()) {
|
||||
throw new RuntimeException("please provide image instruction, current is empty");
|
||||
}
|
||||
|
||||
var instructions = prompt.getInstructions().get(0).getText();
|
||||
var runtimeImageOptions = prompt.getOptions();
|
||||
ImageGenerationOptions imageGenerationOptions = new ImageGenerationOptions(instructions);
|
||||
|
||||
if (this.defaultOptions != null) {
|
||||
// Merge options fixed in beta7
|
||||
// https://github.com/Azure/azure-sdk-for-java/issues/38183
|
||||
runtimeImageOptions = ModelOptionsUtils.merge(runtimeImageOptions, this.defaultOptions,
|
||||
AzureOpenAiImageOptions.class);
|
||||
}
|
||||
|
||||
if (runtimeImageOptions != null) {
|
||||
// Handle portable image options
|
||||
if (runtimeImageOptions.getN() != null) {
|
||||
imageGenerationOptions.setN(runtimeImageOptions.getN());
|
||||
}
|
||||
if (runtimeImageOptions.getModel() != null) {
|
||||
imageGenerationOptions.setModel(runtimeImageOptions.getModel());
|
||||
}
|
||||
if (runtimeImageOptions.getResponseFormat() != null) {
|
||||
// b64_json or url
|
||||
imageGenerationOptions.setResponseFormat(
|
||||
ImageGenerationResponseFormat.fromString(runtimeImageOptions.getResponseFormat()));
|
||||
}
|
||||
if (runtimeImageOptions.getWidth() != null && runtimeImageOptions.getHeight() != null) {
|
||||
imageGenerationOptions.setSize(
|
||||
ImageSize.fromString(runtimeImageOptions.getWidth() + "x" + runtimeImageOptions.getHeight()));
|
||||
}
|
||||
|
||||
// Handle OpenAI specific image options
|
||||
if (runtimeImageOptions instanceof AzureOpenAiImageOptions runtimeAzureOpenAiImageOptions) {
|
||||
if (runtimeAzureOpenAiImageOptions.getQuality() != null) {
|
||||
imageGenerationOptions
|
||||
.setQuality(ImageGenerationQuality.fromString(runtimeAzureOpenAiImageOptions.getQuality()));
|
||||
}
|
||||
if (runtimeAzureOpenAiImageOptions.getStyle() != null) {
|
||||
imageGenerationOptions
|
||||
.setStyle(ImageGenerationStyle.fromString(runtimeAzureOpenAiImageOptions.getStyle()));
|
||||
}
|
||||
if (runtimeAzureOpenAiImageOptions.getUser() != null) {
|
||||
imageGenerationOptions.setUser(runtimeAzureOpenAiImageOptions.getUser());
|
||||
}
|
||||
}
|
||||
}
|
||||
return imageGenerationOptions;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package org.springframework.ai.azure.openai;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import org.springframework.ai.image.ImageOptions;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
/**
|
||||
* The configuration information for a image generation request.
|
||||
*
|
||||
* @author Benoit Moussaud
|
||||
* @since 1.0.0 M1
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class AzureOpenAiImageOptions implements ImageOptions {
|
||||
|
||||
public static final String DEFAULT_IMAGE_MODEL = ImageModel.DALL_E_3.getValue();
|
||||
|
||||
/**
|
||||
* The number of images to generate. Must be between 1 and 10. For dall-e-3, only n=1
|
||||
* is supported.
|
||||
*/
|
||||
@JsonProperty("n")
|
||||
private Integer n;
|
||||
|
||||
/**
|
||||
* The model dall-e-3 or dall-e-2 By default dall-e-3
|
||||
*/
|
||||
@JsonProperty(value = "model")
|
||||
private String model = ImageModel.DALL_E_3.value;
|
||||
|
||||
/**
|
||||
* The deployment name as defined in Azure Open AI Studio when creating a deployment
|
||||
* backed by an Azure OpenAI base model.
|
||||
*/
|
||||
@JsonProperty(value = "deployment_name")
|
||||
private String deploymentName;
|
||||
|
||||
/**
|
||||
* The width of the generated images. Must be one of 256, 512, or 1024 for dall-e-2.
|
||||
*/
|
||||
@JsonProperty("size_width")
|
||||
private Integer width;
|
||||
|
||||
/**
|
||||
* The height of the generated images. Must be one of 256, 512, or 1024 for dall-e-2.
|
||||
*/
|
||||
@JsonProperty("size_height")
|
||||
private Integer height;
|
||||
|
||||
/**
|
||||
* The quality of the image that will be generated. hd creates images with finer
|
||||
* details and greater consistency across the image. This param is only supported for
|
||||
* dall-e-3. standard or hd
|
||||
*/
|
||||
@JsonProperty("quality")
|
||||
private String quality;
|
||||
|
||||
/**
|
||||
* The format in which the generated images are returned. Must be one of url or
|
||||
* b64_json.
|
||||
*/
|
||||
@JsonProperty("response_format")
|
||||
private String responseFormat;
|
||||
|
||||
/**
|
||||
* The size of the generated images. Must be one of 256x256, 512x512, or 1024x1024 for
|
||||
* dall-e-2. Must be one of 1024x1024, 1792x1024, or 1024x1792 for dall-e-3 models.
|
||||
*/
|
||||
@JsonProperty("size")
|
||||
private String size;
|
||||
|
||||
/**
|
||||
* The style of the generated images. Must be one of vivid or natural. Vivid causes
|
||||
* the model to lean towards generating hyper-real and dramatic images. Natural causes
|
||||
* the model to produce more natural, less hyper-real looking images. This param is
|
||||
* only supported for dall-e-3. natural or vivid
|
||||
*/
|
||||
@JsonProperty("style")
|
||||
private String style;
|
||||
|
||||
/**
|
||||
* A unique identifier representing your end-user, which can help OpenAI to monitor
|
||||
* and detect abuse.
|
||||
*/
|
||||
@JsonProperty("user")
|
||||
private String user;
|
||||
|
||||
public Integer getN() {
|
||||
return n;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public void setModel(String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
public void setN(Integer n) {
|
||||
this.n = n;
|
||||
}
|
||||
|
||||
public Integer getWidth() {
|
||||
return width;
|
||||
}
|
||||
|
||||
public void setWidth(Integer width) {
|
||||
this.width = width;
|
||||
this.size = this.width + "x" + this.height;
|
||||
}
|
||||
|
||||
public Integer getHeight() {
|
||||
return height;
|
||||
}
|
||||
|
||||
public void setHeight(Integer height) {
|
||||
this.height = height;
|
||||
this.size = this.width + "x" + this.height;
|
||||
}
|
||||
|
||||
public String getResponseFormat() {
|
||||
return responseFormat;
|
||||
}
|
||||
|
||||
public void setResponseFormat(String responseFormat) {
|
||||
this.responseFormat = responseFormat;
|
||||
}
|
||||
|
||||
public String getSize() {
|
||||
if (this.size != null) {
|
||||
return this.size;
|
||||
}
|
||||
return (this.width != null && this.height != null) ? this.width + "x" + this.height : null;
|
||||
}
|
||||
|
||||
public void setSize(String size) {
|
||||
this.size = size;
|
||||
}
|
||||
|
||||
public String getUser() {
|
||||
return user;
|
||||
}
|
||||
|
||||
public void setUser(String user) {
|
||||
this.user = user;
|
||||
}
|
||||
|
||||
public String getQuality() {
|
||||
return quality;
|
||||
}
|
||||
|
||||
public void setQuality(String quality) {
|
||||
this.quality = quality;
|
||||
}
|
||||
|
||||
public String getStyle() {
|
||||
return style;
|
||||
}
|
||||
|
||||
public void setStyle(String style) {
|
||||
this.style = style;
|
||||
}
|
||||
|
||||
public String getDeploymentName() {
|
||||
return deploymentName;
|
||||
}
|
||||
|
||||
public void setDeploymentName(String deploymentName) {
|
||||
this.deploymentName = deploymentName;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o)
|
||||
return true;
|
||||
if (!(o instanceof AzureOpenAiImageOptions that))
|
||||
return false;
|
||||
return Objects.equals(n, that.n) && Objects.equals(model, that.model)
|
||||
&& Objects.equals(deploymentName, that.deploymentName) && Objects.equals(width, that.width)
|
||||
&& Objects.equals(height, that.height) && Objects.equals(quality, that.quality)
|
||||
&& Objects.equals(responseFormat, that.responseFormat) && Objects.equals(size, that.size)
|
||||
&& Objects.equals(style, that.style) && Objects.equals(user, that.user);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(n, model, deploymentName, width, height, quality, responseFormat, size, style, user);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AzureOpenAiImageOptions{" + "n=" + n + ", model='" + model + '\'' + ", deploymentName='"
|
||||
+ deploymentName + '\'' + ", width=" + width + ", height=" + height + ", quality='" + quality + '\''
|
||||
+ ", responseFormat='" + responseFormat + '\'' + ", size='" + size + '\'' + ", style='" + style + '\''
|
||||
+ ", user='" + user + '\'' + '}';
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
|
||||
private final AzureOpenAiImageOptions options;
|
||||
|
||||
private Builder() {
|
||||
this.options = new AzureOpenAiImageOptions();
|
||||
}
|
||||
|
||||
public Builder withN(Integer n) {
|
||||
options.setN(n);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withModel(String model) {
|
||||
options.setModel(model);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withDeploymentName(String deploymentName) {
|
||||
options.setDeploymentName(deploymentName);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withResponseFormat(String responseFormat) {
|
||||
options.setResponseFormat(responseFormat);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withWidth(Integer width) {
|
||||
options.setWidth(width);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withHeight(Integer height) {
|
||||
options.setHeight(height);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withUser(String user) {
|
||||
options.setUser(user);
|
||||
return this;
|
||||
}
|
||||
|
||||
public AzureOpenAiImageOptions build() {
|
||||
return options;
|
||||
}
|
||||
|
||||
public Builder withStyle(String style) {
|
||||
options.setStyle(style);
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public enum ImageModel {
|
||||
|
||||
/**
|
||||
* The latest DALL·E model released in Nov 2023.
|
||||
*/
|
||||
DALL_E_3("dall-e-3"),
|
||||
|
||||
/**
|
||||
* The previous DALL·E model released in Nov 2022. The 2nd iteration of DALL·E
|
||||
* with more realistic, accurate, and 4x greater resolution images than the
|
||||
* original model.
|
||||
*/
|
||||
DALL_E_2("dall-e-2");
|
||||
|
||||
private final String value;
|
||||
|
||||
ImageModel(String model) {
|
||||
this.value = model;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return this.value;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package org.springframework.ai.azure.openai.metadata;
|
||||
|
||||
import org.springframework.ai.image.ImageGenerationMetadata;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Represents the metadata for image generation using Azure OpenAI.
|
||||
*
|
||||
* @author Benoit Moussaud
|
||||
* @since 1.0.0 M1
|
||||
*/
|
||||
public class AzureOpenAiImageGenerationMetadata implements ImageGenerationMetadata {
|
||||
|
||||
private final String revisedPrompt;
|
||||
|
||||
public AzureOpenAiImageGenerationMetadata(String revisedPrompt) {
|
||||
this.revisedPrompt = revisedPrompt;
|
||||
}
|
||||
|
||||
public String getRevisedPrompt() {
|
||||
return revisedPrompt;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return "AzureOpenAiImageGenerationMetadata{" + "revisedPrompt='" + revisedPrompt + '\'' + '}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o)
|
||||
return true;
|
||||
if (!(o instanceof AzureOpenAiImageGenerationMetadata that))
|
||||
return false;
|
||||
return Objects.equals(revisedPrompt, that.revisedPrompt);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(revisedPrompt);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package org.springframework.ai.azure.openai.metadata;
|
||||
|
||||
import com.azure.ai.openai.models.ImageGenerations;
|
||||
import org.springframework.ai.image.ImageResponseMetadata;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Represents metadata associated with an image response from the Azure OpenAI image
|
||||
* model. It provides additional information about the generative response from the Azure
|
||||
* OpenAI image model, including the creation timestamp of the generated image.
|
||||
*
|
||||
* @author Benoit Moussaud
|
||||
* @since 1.0.0 M1
|
||||
*/
|
||||
public class AzureOpenAiImageResponseMetadata extends HashMap<String, Object> implements ImageResponseMetadata {
|
||||
|
||||
private final Long created;
|
||||
|
||||
public static AzureOpenAiImageResponseMetadata from(ImageGenerations openAiImageResponse) {
|
||||
Assert.notNull(openAiImageResponse, "OpenAiImageResponse must not be null");
|
||||
return new AzureOpenAiImageResponseMetadata(openAiImageResponse.getCreatedAt().toEpochSecond());
|
||||
}
|
||||
|
||||
protected AzureOpenAiImageResponseMetadata(Long created) {
|
||||
this.created = created;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getCreated() {
|
||||
return this.created;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AzureOpenAiImageResponseMetadata{" + "created=" + created + '}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o)
|
||||
return true;
|
||||
if (!(o instanceof AzureOpenAiImageResponseMetadata that))
|
||||
return false;
|
||||
return Objects.equals(created, that.created);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(created);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package org.springframework.ai.azure.openai.image;
|
||||
|
||||
import com.azure.ai.openai.OpenAIClient;
|
||||
import com.azure.ai.openai.OpenAIClientBuilder;
|
||||
import com.azure.core.credential.AzureKeyCredential;
|
||||
import org.assertj.core.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.ai.azure.openai.AzureOpenAiImageModel;
|
||||
import org.springframework.ai.azure.openai.AzureOpenAiImageOptions;
|
||||
import org.springframework.ai.azure.openai.metadata.AzureOpenAiImageGenerationMetadata;
|
||||
import org.springframework.ai.image.*;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringBootConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@SpringBootTest(classes = AzureOpenAiImageModelIT.TestConfiguration.class)
|
||||
@EnabledIfEnvironmentVariable(named = "AZURE_OPENAI_API_KEY", matches = ".+")
|
||||
@EnabledIfEnvironmentVariable(named = "AZURE_OPENAI_ENDPOINT", matches = ".+")
|
||||
public class AzureOpenAiImageModelIT {
|
||||
|
||||
@Autowired
|
||||
protected ImageModel imageModel;
|
||||
|
||||
@Test
|
||||
void imageAsUrlTest() {
|
||||
var options = ImageOptionsBuilder.builder().withHeight(1024).withWidth(1024).build();
|
||||
|
||||
var instructions = """
|
||||
A light cream colored mini golden doodle with a sign that contains the message "I'm on my way to BARCADE!".""";
|
||||
|
||||
ImagePrompt imagePrompt = new ImagePrompt(instructions, options);
|
||||
|
||||
ImageResponse imageResponse = imageModel.call(imagePrompt);
|
||||
|
||||
assertThat(imageResponse.getResults()).hasSize(1);
|
||||
|
||||
ImageResponseMetadata imageResponseMetadata = imageResponse.getMetadata();
|
||||
assertThat(imageResponseMetadata.getCreated()).isPositive();
|
||||
|
||||
var generation = imageResponse.getResult();
|
||||
Image image = generation.getOutput();
|
||||
assertThat(image.getUrl()).isNotEmpty();
|
||||
// System.out.println(image.getUrl());
|
||||
assertThat(image.getB64Json()).isNull();
|
||||
|
||||
var imageGenerationMetadata = generation.getMetadata();
|
||||
Assertions.assertThat(imageGenerationMetadata).isInstanceOf(AzureOpenAiImageGenerationMetadata.class);
|
||||
|
||||
AzureOpenAiImageGenerationMetadata openAiImageGenerationMetadata = (AzureOpenAiImageGenerationMetadata) imageGenerationMetadata;
|
||||
|
||||
assertThat(openAiImageGenerationMetadata).isNotNull();
|
||||
assertThat(openAiImageGenerationMetadata.getRevisedPrompt()).isNotBlank();
|
||||
}
|
||||
|
||||
@SpringBootConfiguration
|
||||
public static class TestConfiguration {
|
||||
|
||||
@Bean
|
||||
public OpenAIClient openAIClient() {
|
||||
return new OpenAIClientBuilder().credential(new AzureKeyCredential(System.getenv("AZURE_OPENAI_API_KEY")))
|
||||
.endpoint(System.getenv("AZURE_OPENAI_ENDPOINT"))
|
||||
.buildClient();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public AzureOpenAiImageModel azureOpenAiImageModel(OpenAIClient openAIClient) {
|
||||
return new AzureOpenAiImageModel(openAIClient,
|
||||
AzureOpenAiImageOptions.builder().withDeploymentName("Dalle3").build());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -44,6 +44,7 @@
|
||||
*** xref:api/embeddings/minimax-embeddings.adoc[MiniMax]
|
||||
*** xref:api/embeddings/zhipuai-embeddings.adoc[ZhiPu AI]
|
||||
** xref:api/imageclient.adoc[]
|
||||
*** xref:api/image/azure-openai-image.adoc[Azure OpenAI]
|
||||
*** xref:api/image/openai-image.adoc[OpenAI]
|
||||
*** xref:api/image/stabilityai-image.adoc[Stability]
|
||||
*** xref:api/image/zhipuai-image.adoc[ZhiPuAI]
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
= Azure OpenAI Image Generation
|
||||
|
||||
|
||||
Spring AI supports DALL-E, the Image generation model from Azure OpenAI.
|
||||
|
||||
== Prerequisites
|
||||
|
||||
Obtain your Azure OpenAI `endpoint` and `api-key` from the Azure OpenAI Service section on the link:https://portal.azure.com[Azure Portal].
|
||||
Spring AI defines a configuration property named `spring.ai.azure.openai.api-key` that you should set to the value of the `API Key` obtained from Azure.
|
||||
There is also a configuration property named `spring.ai.azure.openai.endpoint` that you should set to the endpoint URL obtained when provisioning your model in Azure.
|
||||
Exporting environment variables is one way to set these configuration properties:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
export SPRING_AI_AZURE_OPENAI_API_KEY=<INSERT KEY HERE>
|
||||
export SPRING_AI_AZURE_OPENAI_ENDPOINT=<INSERT ENDPOINT URL HERE>
|
||||
----
|
||||
|
||||
=== Deployment Name
|
||||
|
||||
To use run Azure AI applications, create an Azure AI Deployment through the [Azure AI Portal](https://oai.azure.com/portal).
|
||||
|
||||
In Azure, each client must specify a `Deployment Name` to connect to the Azure OpenAI service.
|
||||
|
||||
It's essential to understand that the `Deployment Name` is different from the model you choose to deploy
|
||||
|
||||
For instance, a deployment named 'MyImgAiDeployment' could be configured to use either the `Dalle3` model or the `Dalle2` model.
|
||||
|
||||
For now, to keep things simple, you can create a deployment using the following settings:
|
||||
|
||||
Deployment Name: `MyImgAiDeployment`
|
||||
Model Name: `Dalle3`
|
||||
|
||||
This Azure configuration will align with the default configurations of the Spring Boot Azure AI Starter and its Autoconfiguration feature.
|
||||
|
||||
If you use a different Deployment Name, update the configuration property accordingly:
|
||||
|
||||
```
|
||||
spring.ai.azure.openai.image.options.deployment-name=<my deployment name>
|
||||
```
|
||||
|
||||
The different deployment structures of Azure OpenAI and OpenAI leads to a property in the Azure OpenAI client library named `deploymentOrModelName`.
|
||||
This is because in OpenAI there is no `Deployment Name`, only a `Model Name`.
|
||||
|
||||
=== Add Repositories and BOM
|
||||
|
||||
Spring AI artifacts are published in Spring Milestone and Snapshot repositories. Refer to the xref:getting-started.adoc#repositories[Repositories] section to add these repositories to your build system.
|
||||
|
||||
To help with dependency management, Spring AI provides a BOM (bill of materials) to ensure that a consistent version of Spring AI is used throughout the entire project. Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build system.
|
||||
|
||||
|
||||
== Auto-configuration
|
||||
|
||||
Spring AI provides Spring Boot auto-configuration for the Azure OpenAI Chat Client.
|
||||
To enable it add the following dependency to your project's Maven `pom.xml` file:
|
||||
|
||||
[source, xml]
|
||||
----
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-azure-openai-spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
----
|
||||
|
||||
or to your Gradle `build.gradle` build file.
|
||||
|
||||
[source,groovy]
|
||||
----
|
||||
dependencies {
|
||||
implementation 'org.springframework.ai:spring-ai-azure-openai-spring-boot-starter'
|
||||
}
|
||||
----
|
||||
|
||||
TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file.
|
||||
|
||||
=== Image Generation Properties
|
||||
|
||||
|
||||
The prefix `spring.ai.openai.image` is the property prefix that lets you configure the `ImageModel` implementation for OpenAI.
|
||||
|
||||
[cols="3,5,1"]
|
||||
|====
|
||||
| Property | Description | Default
|
||||
| spring.ai.azure.openai.image.enabled | Enable OpenAI image model. | true
|
||||
| spring.ai.azure.openai.image.options.n | The number of images to generate. Must be between 1 and 10. For dall-e-3, only n=1 is supported. | -
|
||||
| spring.ai.azure.openai.image.options.model | The model to use for image generation. | AzureOpenAiImageOptions.DEFAULT_IMAGE_MODEL
|
||||
| spring.ai.azure.openai.image.options.quality | The quality of the image that will be generated. HD creates images with finer details and greater consistency across the image. This parameter is only supported for dall-e-3. | -
|
||||
| spring.ai.azure.openai.image.options.response_format | The format in which the generated images are returned. Must be one of URL or b64_json. | -
|
||||
| `spring.ai.openai.image.options.size` | The size of the generated images. Must be one of 256x256, 512x512, or 1024x1024 for dall-e-2. Must be one of 1024x1024, 1792x1024, or 1024x1792 for dall-e-3 models. | -
|
||||
| `spring.ai.openai.image.options.size_width` | The width of the generated images. Must be one of 256, 512, or 1024 for dall-e-2. | -
|
||||
| `spring.ai.openai.image.options.size_height`| The height of the generated images. Must be one of 256, 512, or 1024 for dall-e-2. | -
|
||||
| `spring.ai.openai.image.options.style` | The style of the generated images. Must be one of vivid or natural. Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images. This parameter is only supported for dall-e-3. | -
|
||||
| `spring.ai.openai.image.options.user` | A unique identifier representing your end-user, which can help Azure OpenAI to monitor and detect abuse. | -
|
||||
|====
|
||||
|
||||
==== Connection Properties
|
||||
|
||||
The prefix `spring.ai.openai` is used as the property prefix that lets you connect to Azure OpenAI.
|
||||
|
||||
[cols="3,5,1"]
|
||||
|====
|
||||
| Property | Description | Default
|
||||
| spring.ai.azure.openai.endpoint | The URL to connect to | https://my-dalle3.openai.azure.com/
|
||||
| spring.ai.azure.openai.apiKey | The API Key | -
|
||||
|====
|
||||
|
||||
== Runtime Options [[image-options]]
|
||||
|
||||
The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiImageOptions.java[OpenAiImageOptions.java] provides model configurations, such as the model to use, the quality, the size, etc.
|
||||
|
||||
On start-up, the default options can be configured with the `AzureOpenAiImageModel(OpenAiImageApi openAiImageApi)` constructor and the `withDefaultOptions(OpenAiImageOptions defaultOptions)` method. Alternatively, use the `spring.ai.azure.openai.image.options.*` properties described previously.
|
||||
|
||||
At runtime you can override the default options by adding new, request specific, options to the `ImagePrompt` call.
|
||||
For example to override the OpenAI specific options such as quality and the number of images to create, use the following code example:
|
||||
|
||||
[source,java]
|
||||
----
|
||||
ImageResponse response = azureOpenaiImageModel.call(
|
||||
new ImagePrompt("A light cream colored mini golden doodle",
|
||||
OpenAiImageOptions.builder()
|
||||
.withQuality("hd")
|
||||
.withN(4)
|
||||
.withHeight(1024)
|
||||
.withWidth(1024).build())
|
||||
|
||||
);
|
||||
----
|
||||
|
||||
TIP: In addition to the model specific https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageOptions.java[AzureOpenAiImageOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageOptions.java[ImageOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageOptionsBuilder.java[ImageOptionsBuilder#builder()].
|
||||
@@ -24,6 +24,7 @@ import com.azure.core.util.ClientOptions;
|
||||
|
||||
import org.springframework.ai.azure.openai.AzureOpenAiChatModel;
|
||||
import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel;
|
||||
import org.springframework.ai.azure.openai.AzureOpenAiImageModel;
|
||||
import org.springframework.ai.model.function.FunctionCallback;
|
||||
import org.springframework.ai.model.function.FunctionCallbackContext;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
@@ -39,7 +40,7 @@ import org.springframework.util.CollectionUtils;
|
||||
@AutoConfiguration
|
||||
@ConditionalOnClass({ OpenAIClientBuilder.class, AzureOpenAiChatModel.class })
|
||||
@EnableConfigurationProperties({ AzureOpenAiChatProperties.class, AzureOpenAiEmbeddingProperties.class,
|
||||
AzureOpenAiConnectionProperties.class })
|
||||
AzureOpenAiConnectionProperties.class, AzureOpenAiImageOptionsProperties.class })
|
||||
public class AzureOpenAiAutoConfiguration {
|
||||
|
||||
@Bean
|
||||
@@ -89,4 +90,13 @@ public class AzureOpenAiAutoConfiguration {
|
||||
return manager;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = AzureOpenAiImageOptionsProperties.CONFIG_PREFIX, name = "enabled",
|
||||
havingValue = "true", matchIfMissing = true)
|
||||
public AzureOpenAiImageModel azureOpenAiImageClient(OpenAIClient openAIClient,
|
||||
AzureOpenAiImageOptionsProperties imageProperties) {
|
||||
|
||||
return new AzureOpenAiImageModel(openAIClient, imageProperties.getOptions());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package org.springframework.ai.autoconfigure.azure.openai;
|
||||
|
||||
import org.springframework.ai.azure.openai.AzureOpenAiImageOptions;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
|
||||
/**
|
||||
* Configuration properties for Azure OpenAI image generation options.
|
||||
*
|
||||
* @author Benoit Moussaud
|
||||
* @since 1.0.0 M1
|
||||
*/
|
||||
@ConfigurationProperties(AzureOpenAiImageOptionsProperties.CONFIG_PREFIX)
|
||||
public class AzureOpenAiImageOptionsProperties {
|
||||
|
||||
public static final String CONFIG_PREFIX = "spring.ai.azure.openai.image";
|
||||
|
||||
/**
|
||||
* Enable Azure OpenAI chat client.
|
||||
*/
|
||||
private boolean enabled = true;
|
||||
|
||||
@NestedConfigurationProperty
|
||||
private AzureOpenAiImageOptions options = AzureOpenAiImageOptions.builder().build();
|
||||
|
||||
public AzureOpenAiImageOptions getOptions() {
|
||||
return options;
|
||||
}
|
||||
|
||||
public void setOptions(AzureOpenAiImageOptions options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user