From a30cc9d84df1486369c23d422c35b012e2b23735 Mon Sep 17 00:00:00 2001 From: Benoit Moussaud Date: Mon, 27 May 2024 15:16:19 +0200 Subject: [PATCH] Add the support of Azure OpenAI image generation * Add docs --- .../azure/openai/AzureOpenAiImageModel.java | 190 ++++++++++++ .../azure/openai/AzureOpenAiImageOptions.java | 287 ++++++++++++++++++ .../AzureOpenAiImageGenerationMetadata.java | 43 +++ .../AzureOpenAiImageResponseMetadata.java | 55 ++++ .../openai/image/AzureOpenAiImageModelIT.java | 79 +++++ .../src/main/antora/modules/ROOT/nav.adoc | 1 + .../pages/api/image/azure-openai-image.adoc | 129 ++++++++ .../openai/AzureOpenAiAutoConfiguration.java | 12 +- .../AzureOpenAiImageOptionsProperties.java | 42 +++ 9 files changed, 837 insertions(+), 1 deletion(-) create mode 100644 models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageModel.java create mode 100644 models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageOptions.java create mode 100644 models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/metadata/AzureOpenAiImageGenerationMetadata.java create mode 100644 models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/metadata/AzureOpenAiImageResponseMetadata.java create mode 100644 models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/image/AzureOpenAiImageModelIT.java create mode 100644 spring-ai-docs/src/main/antora/modules/ROOT/pages/api/image/azure-openai-image.adoc create mode 100644 spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/azure/openai/AzureOpenAiImageOptionsProperties.java diff --git a/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageModel.java b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageModel.java new file mode 100644 index 000000000..e6da1ebbf --- /dev/null +++ b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageModel.java @@ -0,0 +1,190 @@ +package org.springframework.ai.azure.openai; + +import com.azure.ai.openai.OpenAIClient; +import com.azure.ai.openai.models.ImageGenerationOptions; +import com.azure.ai.openai.models.ImageGenerationQuality; +import com.azure.ai.openai.models.ImageGenerationResponseFormat; +import com.azure.ai.openai.models.ImageGenerationStyle; +import com.azure.ai.openai.models.ImageSize; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.azure.openai.metadata.AzureOpenAiImageGenerationMetadata; +import org.springframework.ai.azure.openai.metadata.AzureOpenAiImageResponseMetadata; +import org.springframework.ai.image.Image; +import org.springframework.ai.image.ImageGeneration; +import org.springframework.ai.image.ImageModel; +import org.springframework.ai.image.ImagePrompt; +import org.springframework.ai.image.ImageResponse; +import org.springframework.ai.image.ImageResponseMetadata; +import org.springframework.ai.model.ModelOptionsUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.Assert; + +import java.util.List; + +import static java.lang.String.format; + +/** + * {@link ImageModel} implementation for {@literal Microsoft Azure AI} backed by + * {@link OpenAIClient}. + * + * @author Benoit Moussaud + * @see ImageModel + * @see com.azure.ai.openai.OpenAIClient + * @since 1.0.0 M1 + */ +public class AzureOpenAiImageModel implements ImageModel { + + private static final String DEFAULT_DEPLOYMENT_NAME = AzureOpenAiImageOptions.DEFAULT_IMAGE_MODEL; + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Autowired + private final OpenAIClient openAIClient; + + private final AzureOpenAiImageOptions defaultOptions; + + public AzureOpenAiImageModel(OpenAIClient openAIClient) { + this(openAIClient, AzureOpenAiImageOptions.builder().withDeploymentName(DEFAULT_DEPLOYMENT_NAME).build()); + } + + public AzureOpenAiImageModel(OpenAIClient microsoftOpenAiClient, AzureOpenAiImageOptions options) { + Assert.notNull(microsoftOpenAiClient, "com.azure.ai.openai.OpenAIClient must not be null"); + Assert.notNull(options, "AzureOpenAiChatOptions must not be null"); + this.openAIClient = microsoftOpenAiClient; + this.defaultOptions = options; + } + + public AzureOpenAiImageOptions getDefaultOptions() { + return defaultOptions; + } + + @Override + public ImageResponse call(ImagePrompt imagePrompt) { + ImageGenerationOptions imageGenerationOptions = toOpenAiImageOptions(imagePrompt); + String deploymentOrModelName = getDeploymentName(imagePrompt); + if (logger.isTraceEnabled()) { + logger.trace("Azure ImageGenerationOptions call {} with the following options : {} ", deploymentOrModelName, + toPrettyJson(imageGenerationOptions)); + } + + var images = openAIClient.getImageGenerations(deploymentOrModelName, imageGenerationOptions); + + if (logger.isTraceEnabled()) { + logger.trace("Azure ImageGenerations: {}", toPrettyJson(images)); + } + + List imageGenerations = images.getData().stream().map(entry -> { + var image = new Image(entry.getUrl(), entry.getBase64Data()); + var metadata = new AzureOpenAiImageGenerationMetadata(entry.getRevisedPrompt()); + return new ImageGeneration(image, metadata); + }).toList(); + + ImageResponseMetadata openAiImageResponseMetadata = AzureOpenAiImageResponseMetadata.from(images); + return new ImageResponse(imageGenerations, openAiImageResponseMetadata); + } + + private String toPrettyJson(Object object) { + ObjectMapper objectMapper = new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + .registerModule(new JavaTimeModule()); + try { + return objectMapper.writeValueAsString(object); + } + catch (JsonProcessingException e) { + return "JsonProcessingException:" + e + " [" + object.toString() + "]"; + } + } + + /** + * Return the deployment-name if provided or use the model name. + * @param prompt the image prompt + * @return Return the deployment-name if provided or use the model name. + */ + private String getDeploymentName(ImagePrompt prompt) { + var runtimeImageOptions = prompt.getOptions(); + + if (this.defaultOptions != null) { + // Merge options fixed in beta7 + // https://github.com/Azure/azure-sdk-for-java/issues/38183 + runtimeImageOptions = ModelOptionsUtils.merge(runtimeImageOptions, this.defaultOptions, + AzureOpenAiImageOptions.class); + } + + if (runtimeImageOptions != null) { + if (runtimeImageOptions instanceof AzureOpenAiImageOptions runtimeAzureOpenAiImageOptions) { + if (runtimeAzureOpenAiImageOptions.getDeploymentName() != null) { + return runtimeAzureOpenAiImageOptions.getDeploymentName(); + } + } + + } + + // By default the one provided in the image prompt + return prompt.getOptions().getModel(); + + } + + private ImageGenerationOptions toOpenAiImageOptions(ImagePrompt prompt) { + + if (prompt.getInstructions().size() > 1) { + throw new RuntimeException(format("implementation support 1 image instruction only, found %s", + prompt.getInstructions().size())); + } + if (prompt.getInstructions().isEmpty()) { + throw new RuntimeException("please provide image instruction, current is empty"); + } + + var instructions = prompt.getInstructions().get(0).getText(); + var runtimeImageOptions = prompt.getOptions(); + ImageGenerationOptions imageGenerationOptions = new ImageGenerationOptions(instructions); + + if (this.defaultOptions != null) { + // Merge options fixed in beta7 + // https://github.com/Azure/azure-sdk-for-java/issues/38183 + runtimeImageOptions = ModelOptionsUtils.merge(runtimeImageOptions, this.defaultOptions, + AzureOpenAiImageOptions.class); + } + + if (runtimeImageOptions != null) { + // Handle portable image options + if (runtimeImageOptions.getN() != null) { + imageGenerationOptions.setN(runtimeImageOptions.getN()); + } + if (runtimeImageOptions.getModel() != null) { + imageGenerationOptions.setModel(runtimeImageOptions.getModel()); + } + if (runtimeImageOptions.getResponseFormat() != null) { + // b64_json or url + imageGenerationOptions.setResponseFormat( + ImageGenerationResponseFormat.fromString(runtimeImageOptions.getResponseFormat())); + } + if (runtimeImageOptions.getWidth() != null && runtimeImageOptions.getHeight() != null) { + imageGenerationOptions.setSize( + ImageSize.fromString(runtimeImageOptions.getWidth() + "x" + runtimeImageOptions.getHeight())); + } + + // Handle OpenAI specific image options + if (runtimeImageOptions instanceof AzureOpenAiImageOptions runtimeAzureOpenAiImageOptions) { + if (runtimeAzureOpenAiImageOptions.getQuality() != null) { + imageGenerationOptions + .setQuality(ImageGenerationQuality.fromString(runtimeAzureOpenAiImageOptions.getQuality())); + } + if (runtimeAzureOpenAiImageOptions.getStyle() != null) { + imageGenerationOptions + .setStyle(ImageGenerationStyle.fromString(runtimeAzureOpenAiImageOptions.getStyle())); + } + if (runtimeAzureOpenAiImageOptions.getUser() != null) { + imageGenerationOptions.setUser(runtimeAzureOpenAiImageOptions.getUser()); + } + } + } + return imageGenerationOptions; + } + +} diff --git a/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageOptions.java b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageOptions.java new file mode 100644 index 000000000..064551556 --- /dev/null +++ b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageOptions.java @@ -0,0 +1,287 @@ +package org.springframework.ai.azure.openai; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonInclude; +import org.springframework.ai.image.ImageOptions; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * The configuration information for a image generation request. + * + * @author Benoit Moussaud + * @since 1.0.0 M1 + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class AzureOpenAiImageOptions implements ImageOptions { + + public static final String DEFAULT_IMAGE_MODEL = ImageModel.DALL_E_3.getValue(); + + /** + * The number of images to generate. Must be between 1 and 10. For dall-e-3, only n=1 + * is supported. + */ + @JsonProperty("n") + private Integer n; + + /** + * The model dall-e-3 or dall-e-2 By default dall-e-3 + */ + @JsonProperty(value = "model") + private String model = ImageModel.DALL_E_3.value; + + /** + * The deployment name as defined in Azure Open AI Studio when creating a deployment + * backed by an Azure OpenAI base model. + */ + @JsonProperty(value = "deployment_name") + private String deploymentName; + + /** + * The width of the generated images. Must be one of 256, 512, or 1024 for dall-e-2. + */ + @JsonProperty("size_width") + private Integer width; + + /** + * The height of the generated images. Must be one of 256, 512, or 1024 for dall-e-2. + */ + @JsonProperty("size_height") + private Integer height; + + /** + * The quality of the image that will be generated. hd creates images with finer + * details and greater consistency across the image. This param is only supported for + * dall-e-3. standard or hd + */ + @JsonProperty("quality") + private String quality; + + /** + * The format in which the generated images are returned. Must be one of url or + * b64_json. + */ + @JsonProperty("response_format") + private String responseFormat; + + /** + * The size of the generated images. Must be one of 256x256, 512x512, or 1024x1024 for + * dall-e-2. Must be one of 1024x1024, 1792x1024, or 1024x1792 for dall-e-3 models. + */ + @JsonProperty("size") + private String size; + + /** + * The style of the generated images. Must be one of vivid or natural. Vivid causes + * the model to lean towards generating hyper-real and dramatic images. Natural causes + * the model to produce more natural, less hyper-real looking images. This param is + * only supported for dall-e-3. natural or vivid + */ + @JsonProperty("style") + private String style; + + /** + * A unique identifier representing your end-user, which can help OpenAI to monitor + * and detect abuse. + */ + @JsonProperty("user") + private String user; + + public Integer getN() { + return n; + } + + @Override + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + public void setN(Integer n) { + this.n = n; + } + + public Integer getWidth() { + return width; + } + + public void setWidth(Integer width) { + this.width = width; + this.size = this.width + "x" + this.height; + } + + public Integer getHeight() { + return height; + } + + public void setHeight(Integer height) { + this.height = height; + this.size = this.width + "x" + this.height; + } + + public String getResponseFormat() { + return responseFormat; + } + + public void setResponseFormat(String responseFormat) { + this.responseFormat = responseFormat; + } + + public String getSize() { + if (this.size != null) { + return this.size; + } + return (this.width != null && this.height != null) ? this.width + "x" + this.height : null; + } + + public void setSize(String size) { + this.size = size; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getQuality() { + return quality; + } + + public void setQuality(String quality) { + this.quality = quality; + } + + public String getStyle() { + return style; + } + + public void setStyle(String style) { + this.style = style; + } + + public String getDeploymentName() { + return deploymentName; + } + + public void setDeploymentName(String deploymentName) { + this.deploymentName = deploymentName; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof AzureOpenAiImageOptions that)) + return false; + return Objects.equals(n, that.n) && Objects.equals(model, that.model) + && Objects.equals(deploymentName, that.deploymentName) && Objects.equals(width, that.width) + && Objects.equals(height, that.height) && Objects.equals(quality, that.quality) + && Objects.equals(responseFormat, that.responseFormat) && Objects.equals(size, that.size) + && Objects.equals(style, that.style) && Objects.equals(user, that.user); + } + + @Override + public int hashCode() { + return Objects.hash(n, model, deploymentName, width, height, quality, responseFormat, size, style, user); + } + + @Override + public String toString() { + return "AzureOpenAiImageOptions{" + "n=" + n + ", model='" + model + '\'' + ", deploymentName='" + + deploymentName + '\'' + ", width=" + width + ", height=" + height + ", quality='" + quality + '\'' + + ", responseFormat='" + responseFormat + '\'' + ", size='" + size + '\'' + ", style='" + style + '\'' + + ", user='" + user + '\'' + '}'; + } + + public static class Builder { + + private final AzureOpenAiImageOptions options; + + private Builder() { + this.options = new AzureOpenAiImageOptions(); + } + + public Builder withN(Integer n) { + options.setN(n); + return this; + } + + public Builder withModel(String model) { + options.setModel(model); + return this; + } + + public Builder withDeploymentName(String deploymentName) { + options.setDeploymentName(deploymentName); + return this; + } + + public Builder withResponseFormat(String responseFormat) { + options.setResponseFormat(responseFormat); + return this; + } + + public Builder withWidth(Integer width) { + options.setWidth(width); + return this; + } + + public Builder withHeight(Integer height) { + options.setHeight(height); + return this; + } + + public Builder withUser(String user) { + options.setUser(user); + return this; + } + + public AzureOpenAiImageOptions build() { + return options; + } + + public Builder withStyle(String style) { + options.setStyle(style); + return this; + } + + } + + public enum ImageModel { + + /** + * The latest DALL·E model released in Nov 2023. + */ + DALL_E_3("dall-e-3"), + + /** + * The previous DALL·E model released in Nov 2022. The 2nd iteration of DALL·E + * with more realistic, accurate, and 4x greater resolution images than the + * original model. + */ + DALL_E_2("dall-e-2"); + + private final String value; + + ImageModel(String model) { + this.value = model; + } + + public String getValue() { + return this.value; + } + + } + +} diff --git a/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/metadata/AzureOpenAiImageGenerationMetadata.java b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/metadata/AzureOpenAiImageGenerationMetadata.java new file mode 100644 index 000000000..44b429e9f --- /dev/null +++ b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/metadata/AzureOpenAiImageGenerationMetadata.java @@ -0,0 +1,43 @@ +package org.springframework.ai.azure.openai.metadata; + +import org.springframework.ai.image.ImageGenerationMetadata; + +import java.util.Objects; + +/** + * Represents the metadata for image generation using Azure OpenAI. + * + * @author Benoit Moussaud + * @since 1.0.0 M1 + */ +public class AzureOpenAiImageGenerationMetadata implements ImageGenerationMetadata { + + private final String revisedPrompt; + + public AzureOpenAiImageGenerationMetadata(String revisedPrompt) { + this.revisedPrompt = revisedPrompt; + } + + public String getRevisedPrompt() { + return revisedPrompt; + } + + public String toString() { + return "AzureOpenAiImageGenerationMetadata{" + "revisedPrompt='" + revisedPrompt + '\'' + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof AzureOpenAiImageGenerationMetadata that)) + return false; + return Objects.equals(revisedPrompt, that.revisedPrompt); + } + + @Override + public int hashCode() { + return Objects.hash(revisedPrompt); + } + +} diff --git a/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/metadata/AzureOpenAiImageResponseMetadata.java b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/metadata/AzureOpenAiImageResponseMetadata.java new file mode 100644 index 000000000..e821913f7 --- /dev/null +++ b/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/metadata/AzureOpenAiImageResponseMetadata.java @@ -0,0 +1,55 @@ +package org.springframework.ai.azure.openai.metadata; + +import com.azure.ai.openai.models.ImageGenerations; +import org.springframework.ai.image.ImageResponseMetadata; +import org.springframework.util.Assert; + +import java.util.HashMap; +import java.util.Objects; + +/** + * Represents metadata associated with an image response from the Azure OpenAI image + * model. It provides additional information about the generative response from the Azure + * OpenAI image model, including the creation timestamp of the generated image. + * + * @author Benoit Moussaud + * @since 1.0.0 M1 + */ +public class AzureOpenAiImageResponseMetadata extends HashMap implements ImageResponseMetadata { + + private final Long created; + + public static AzureOpenAiImageResponseMetadata from(ImageGenerations openAiImageResponse) { + Assert.notNull(openAiImageResponse, "OpenAiImageResponse must not be null"); + return new AzureOpenAiImageResponseMetadata(openAiImageResponse.getCreatedAt().toEpochSecond()); + } + + protected AzureOpenAiImageResponseMetadata(Long created) { + this.created = created; + } + + @Override + public Long getCreated() { + return this.created; + } + + @Override + public String toString() { + return "AzureOpenAiImageResponseMetadata{" + "created=" + created + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof AzureOpenAiImageResponseMetadata that)) + return false; + return Objects.equals(created, that.created); + } + + @Override + public int hashCode() { + return Objects.hash(created); + } + +} diff --git a/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/image/AzureOpenAiImageModelIT.java b/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/image/AzureOpenAiImageModelIT.java new file mode 100644 index 000000000..4aff0c673 --- /dev/null +++ b/models/spring-ai-azure-openai/src/test/java/org/springframework/ai/azure/openai/image/AzureOpenAiImageModelIT.java @@ -0,0 +1,79 @@ +package org.springframework.ai.azure.openai.image; + +import com.azure.ai.openai.OpenAIClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.springframework.ai.azure.openai.AzureOpenAiImageModel; +import org.springframework.ai.azure.openai.AzureOpenAiImageOptions; +import org.springframework.ai.azure.openai.metadata.AzureOpenAiImageGenerationMetadata; +import org.springframework.ai.image.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.util.StringUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(classes = AzureOpenAiImageModelIT.TestConfiguration.class) +@EnabledIfEnvironmentVariable(named = "AZURE_OPENAI_API_KEY", matches = ".+") +@EnabledIfEnvironmentVariable(named = "AZURE_OPENAI_ENDPOINT", matches = ".+") +public class AzureOpenAiImageModelIT { + + @Autowired + protected ImageModel imageModel; + + @Test + void imageAsUrlTest() { + var options = ImageOptionsBuilder.builder().withHeight(1024).withWidth(1024).build(); + + var instructions = """ + A light cream colored mini golden doodle with a sign that contains the message "I'm on my way to BARCADE!"."""; + + ImagePrompt imagePrompt = new ImagePrompt(instructions, options); + + ImageResponse imageResponse = imageModel.call(imagePrompt); + + assertThat(imageResponse.getResults()).hasSize(1); + + ImageResponseMetadata imageResponseMetadata = imageResponse.getMetadata(); + assertThat(imageResponseMetadata.getCreated()).isPositive(); + + var generation = imageResponse.getResult(); + Image image = generation.getOutput(); + assertThat(image.getUrl()).isNotEmpty(); + // System.out.println(image.getUrl()); + assertThat(image.getB64Json()).isNull(); + + var imageGenerationMetadata = generation.getMetadata(); + Assertions.assertThat(imageGenerationMetadata).isInstanceOf(AzureOpenAiImageGenerationMetadata.class); + + AzureOpenAiImageGenerationMetadata openAiImageGenerationMetadata = (AzureOpenAiImageGenerationMetadata) imageGenerationMetadata; + + assertThat(openAiImageGenerationMetadata).isNotNull(); + assertThat(openAiImageGenerationMetadata.getRevisedPrompt()).isNotBlank(); + } + + @SpringBootConfiguration + public static class TestConfiguration { + + @Bean + public OpenAIClient openAIClient() { + return new OpenAIClientBuilder().credential(new AzureKeyCredential(System.getenv("AZURE_OPENAI_API_KEY"))) + .endpoint(System.getenv("AZURE_OPENAI_ENDPOINT")) + .buildClient(); + } + + @Bean + public AzureOpenAiImageModel azureOpenAiImageModel(OpenAIClient openAIClient) { + return new AzureOpenAiImageModel(openAIClient, + AzureOpenAiImageOptions.builder().withDeploymentName("Dalle3").build()); + + } + + } + +} diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc index 397ac6d40..120a8e5c8 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc @@ -44,6 +44,7 @@ *** xref:api/embeddings/minimax-embeddings.adoc[MiniMax] *** xref:api/embeddings/zhipuai-embeddings.adoc[ZhiPu AI] ** xref:api/imageclient.adoc[] +*** xref:api/image/azure-openai-image.adoc[Azure OpenAI] *** xref:api/image/openai-image.adoc[OpenAI] *** xref:api/image/stabilityai-image.adoc[Stability] *** xref:api/image/zhipuai-image.adoc[ZhiPuAI] diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/image/azure-openai-image.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/image/azure-openai-image.adoc new file mode 100644 index 000000000..1485b739c --- /dev/null +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/image/azure-openai-image.adoc @@ -0,0 +1,129 @@ += Azure OpenAI Image Generation + + +Spring AI supports DALL-E, the Image generation model from Azure OpenAI. + +== Prerequisites + +Obtain your Azure OpenAI `endpoint` and `api-key` from the Azure OpenAI Service section on the link:https://portal.azure.com[Azure Portal]. +Spring AI defines a configuration property named `spring.ai.azure.openai.api-key` that you should set to the value of the `API Key` obtained from Azure. +There is also a configuration property named `spring.ai.azure.openai.endpoint` that you should set to the endpoint URL obtained when provisioning your model in Azure. +Exporting environment variables is one way to set these configuration properties: + +[source,shell] +---- +export SPRING_AI_AZURE_OPENAI_API_KEY= +export SPRING_AI_AZURE_OPENAI_ENDPOINT= +---- + +=== Deployment Name + +To use run Azure AI applications, create an Azure AI Deployment through the [Azure AI Portal](https://oai.azure.com/portal). + +In Azure, each client must specify a `Deployment Name` to connect to the Azure OpenAI service. + +It's essential to understand that the `Deployment Name` is different from the model you choose to deploy + +For instance, a deployment named 'MyImgAiDeployment' could be configured to use either the `Dalle3` model or the `Dalle2` model. + +For now, to keep things simple, you can create a deployment using the following settings: + +Deployment Name: `MyImgAiDeployment` +Model Name: `Dalle3` + +This Azure configuration will align with the default configurations of the Spring Boot Azure AI Starter and its Autoconfiguration feature. + +If you use a different Deployment Name, update the configuration property accordingly: + +``` +spring.ai.azure.openai.image.options.deployment-name= +``` + +The different deployment structures of Azure OpenAI and OpenAI leads to a property in the Azure OpenAI client library named `deploymentOrModelName`. +This is because in OpenAI there is no `Deployment Name`, only a `Model Name`. + +=== Add Repositories and BOM + +Spring AI artifacts are published in Spring Milestone and Snapshot repositories. Refer to the xref:getting-started.adoc#repositories[Repositories] section to add these repositories to your build system. + +To help with dependency management, Spring AI provides a BOM (bill of materials) to ensure that a consistent version of Spring AI is used throughout the entire project. Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build system. + + +== Auto-configuration + +Spring AI provides Spring Boot auto-configuration for the Azure OpenAI Chat Client. +To enable it add the following dependency to your project's Maven `pom.xml` file: + +[source, xml] +---- + + org.springframework.ai + spring-ai-azure-openai-spring-boot-starter + +---- + +or to your Gradle `build.gradle` build file. + +[source,groovy] +---- +dependencies { + implementation 'org.springframework.ai:spring-ai-azure-openai-spring-boot-starter' +} +---- + +TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file. + +=== Image Generation Properties + + +The prefix `spring.ai.openai.image` is the property prefix that lets you configure the `ImageModel` implementation for OpenAI. + +[cols="3,5,1"] +|==== +| Property | Description | Default +| spring.ai.azure.openai.image.enabled | Enable OpenAI image model. | true +| spring.ai.azure.openai.image.options.n | The number of images to generate. Must be between 1 and 10. For dall-e-3, only n=1 is supported. | - +| spring.ai.azure.openai.image.options.model | The model to use for image generation. | AzureOpenAiImageOptions.DEFAULT_IMAGE_MODEL +| spring.ai.azure.openai.image.options.quality | The quality of the image that will be generated. HD creates images with finer details and greater consistency across the image. This parameter is only supported for dall-e-3. | - +| spring.ai.azure.openai.image.options.response_format | The format in which the generated images are returned. Must be one of URL or b64_json. | - +| `spring.ai.openai.image.options.size` | The size of the generated images. Must be one of 256x256, 512x512, or 1024x1024 for dall-e-2. Must be one of 1024x1024, 1792x1024, or 1024x1792 for dall-e-3 models. | - +| `spring.ai.openai.image.options.size_width` | The width of the generated images. Must be one of 256, 512, or 1024 for dall-e-2. | - +| `spring.ai.openai.image.options.size_height`| The height of the generated images. Must be one of 256, 512, or 1024 for dall-e-2. | - +| `spring.ai.openai.image.options.style` | The style of the generated images. Must be one of vivid or natural. Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images. This parameter is only supported for dall-e-3. | - +| `spring.ai.openai.image.options.user` | A unique identifier representing your end-user, which can help Azure OpenAI to monitor and detect abuse. | - +|==== + +==== Connection Properties + +The prefix `spring.ai.openai` is used as the property prefix that lets you connect to Azure OpenAI. + +[cols="3,5,1"] +|==== +| Property | Description | Default +| spring.ai.azure.openai.endpoint | The URL to connect to | https://my-dalle3.openai.azure.com/ +| spring.ai.azure.openai.apiKey | The API Key | - +|==== + +== Runtime Options [[image-options]] + +The https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiImageOptions.java[OpenAiImageOptions.java] provides model configurations, such as the model to use, the quality, the size, etc. + +On start-up, the default options can be configured with the `AzureOpenAiImageModel(OpenAiImageApi openAiImageApi)` constructor and the `withDefaultOptions(OpenAiImageOptions defaultOptions)` method. Alternatively, use the `spring.ai.azure.openai.image.options.*` properties described previously. + +At runtime you can override the default options by adding new, request specific, options to the `ImagePrompt` call. +For example to override the OpenAI specific options such as quality and the number of images to create, use the following code example: + +[source,java] +---- +ImageResponse response = azureOpenaiImageModel.call( + new ImagePrompt("A light cream colored mini golden doodle", + OpenAiImageOptions.builder() + .withQuality("hd") + .withN(4) + .withHeight(1024) + .withWidth(1024).build()) + +); +---- + +TIP: In addition to the model specific https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiImageOptions.java[AzureOpenAiImageOptions] you can use a portable https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageOptions.java[ImageOptions] instance, created with the https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/image/ImageOptionsBuilder.java[ImageOptionsBuilder#builder()]. diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/azure/openai/AzureOpenAiAutoConfiguration.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/azure/openai/AzureOpenAiAutoConfiguration.java index b4fb7ed4f..8b0394450 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/azure/openai/AzureOpenAiAutoConfiguration.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/azure/openai/AzureOpenAiAutoConfiguration.java @@ -24,6 +24,7 @@ import com.azure.core.util.ClientOptions; import org.springframework.ai.azure.openai.AzureOpenAiChatModel; import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel; +import org.springframework.ai.azure.openai.AzureOpenAiImageModel; import org.springframework.ai.model.function.FunctionCallback; import org.springframework.ai.model.function.FunctionCallbackContext; import org.springframework.boot.autoconfigure.AutoConfiguration; @@ -39,7 +40,7 @@ import org.springframework.util.CollectionUtils; @AutoConfiguration @ConditionalOnClass({ OpenAIClientBuilder.class, AzureOpenAiChatModel.class }) @EnableConfigurationProperties({ AzureOpenAiChatProperties.class, AzureOpenAiEmbeddingProperties.class, - AzureOpenAiConnectionProperties.class }) + AzureOpenAiConnectionProperties.class, AzureOpenAiImageOptionsProperties.class }) public class AzureOpenAiAutoConfiguration { @Bean @@ -89,4 +90,13 @@ public class AzureOpenAiAutoConfiguration { return manager; } + @Bean + @ConditionalOnProperty(prefix = AzureOpenAiImageOptionsProperties.CONFIG_PREFIX, name = "enabled", + havingValue = "true", matchIfMissing = true) + public AzureOpenAiImageModel azureOpenAiImageClient(OpenAIClient openAIClient, + AzureOpenAiImageOptionsProperties imageProperties) { + + return new AzureOpenAiImageModel(openAIClient, imageProperties.getOptions()); + } + } diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/azure/openai/AzureOpenAiImageOptionsProperties.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/azure/openai/AzureOpenAiImageOptionsProperties.java new file mode 100644 index 000000000..26e1ae2c8 --- /dev/null +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/azure/openai/AzureOpenAiImageOptionsProperties.java @@ -0,0 +1,42 @@ +package org.springframework.ai.autoconfigure.azure.openai; + +import org.springframework.ai.azure.openai.AzureOpenAiImageOptions; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +/** + * Configuration properties for Azure OpenAI image generation options. + * + * @author Benoit Moussaud + * @since 1.0.0 M1 + */ +@ConfigurationProperties(AzureOpenAiImageOptionsProperties.CONFIG_PREFIX) +public class AzureOpenAiImageOptionsProperties { + + public static final String CONFIG_PREFIX = "spring.ai.azure.openai.image"; + + /** + * Enable Azure OpenAI chat client. + */ + private boolean enabled = true; + + @NestedConfigurationProperty + private AzureOpenAiImageOptions options = AzureOpenAiImageOptions.builder().build(); + + public AzureOpenAiImageOptions getOptions() { + return options; + } + + public void setOptions(AzureOpenAiImageOptions options) { + this.options = options; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + +}