diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java index 7d0ba4e65..ff0d28cc2 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java @@ -64,6 +64,7 @@ import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletion; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletion.Choice; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.AudioOutput; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.ChatCompletionFunction; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.MediaContent; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.ToolCall; @@ -72,6 +73,8 @@ import org.springframework.ai.openai.api.common.OpenAiApiConstants; import org.springframework.ai.openai.metadata.OpenAiUsage; import org.springframework.ai.openai.metadata.support.OpenAiResponseHeaderExtractor; import org.springframework.ai.retry.RetryUtils; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; import org.springframework.retry.support.RetryTemplate; import org.springframework.util.Assert; @@ -251,7 +254,7 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode "finishReason", choice.finishReason() != null ? choice.finishReason().name() : "", "refusal", StringUtils.hasText(choice.message().refusal()) ? choice.message().refusal() : ""); // @formatter:on - return buildGeneration(choice, metadata); + return buildGeneration(choice, metadata, request); }).toList(); // Non function calling. @@ -282,6 +285,17 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode return Flux.deferContextual(contextView -> { ChatCompletionRequest request = createRequest(prompt, true); + if (request.outputModalities() != null) { + if (request.outputModalities().stream().anyMatch(m -> m.equals("audio"))) { + logger.warn("Audio output is not supported for streaming requests. Removing audio output."); + throw new IllegalArgumentException("Audio output is not supported for streaming requests."); + } + } + if (request.audioParameters() != null) { + logger.warn("Audio parameters are not supported for streaming requests. Removing audio parameters."); + throw new IllegalArgumentException("Audio parameters are not supported for streaming requests."); + } + Flux completionChunks = this.openAiApi.chatCompletionStream(request, getAdditionalHttpHeaders(prompt)); @@ -320,7 +334,7 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode "finishReason", choice.finishReason() != null ? choice.finishReason().name() : "", "refusal", StringUtils.hasText(choice.message().refusal()) ? choice.message().refusal() : ""); - return buildGeneration(choice, metadata); + return buildGeneration(choice, metadata, request); }).toList(); // @formatter:on @@ -367,7 +381,7 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode headers.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> List.of(e.getValue())))); } - private Generation buildGeneration(Choice choice, Map metadata) { + private Generation buildGeneration(Choice choice, Map metadata, ChatCompletionRequest request) { List toolCalls = choice.message().toolCalls() == null ? List.of() : choice.message() .toolCalls() @@ -376,10 +390,26 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode toolCall.function().name(), toolCall.function().arguments())) .toList(); - var assistantMessage = new AssistantMessage(choice.message().content(), metadata, toolCalls); String finishReason = (choice.finishReason() != null ? choice.finishReason().name() : ""); - var generationMetadata = ChatGenerationMetadata.builder().finishReason(finishReason).build(); - return new Generation(assistantMessage, generationMetadata); + var generationMetadataBuilder = ChatGenerationMetadata.builder().finishReason(finishReason); + + List media = new ArrayList<>(); + String textContent = choice.message().content(); + var audioOutput = choice.message().audioOutput(); + if (audioOutput != null) { + String mimeType = String.format("audio/%s", request.audioParameters().format().name().toLowerCase()); + byte[] audioData = Base64.getDecoder().decode(audioOutput.data()); + Resource resource = new ByteArrayResource(audioData); + media.add(new Media(MimeTypeUtils.parseMimeType(mimeType), resource, audioOutput.id())); + if (!StringUtils.hasText(textContent)) { + textContent = audioOutput.transcript(); + } + generationMetadataBuilder.metadata("audioId", audioOutput.id()); + generationMetadataBuilder.metadata("audioExpiresAt", audioOutput.expiresAt()); + } + + var assistantMessage = new AssistantMessage(textContent, metadata, toolCalls, media); + return new Generation(assistantMessage, generationMetadataBuilder.build()); } private ChatResponseMetadata from(OpenAiApi.ChatCompletion result, RateLimit rateLimit) { @@ -443,8 +473,15 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode return new ToolCall(toolCall.id(), toolCall.type(), function); }).toList(); } + AudioOutput audioOutput = null; + if (!CollectionUtils.isEmpty(assistantMessage.getMedia())) { + Assert.isTrue(assistantMessage.getMedia().size() == 1, + "Only one media content is supported for assistant messages"); + audioOutput = new AudioOutput(assistantMessage.getMedia().get(0).getId(), null, null, null); + + } return List.of(new ChatCompletionMessage(assistantMessage.getContent(), - ChatCompletionMessage.Role.ASSISTANT, null, null, toolCalls, null, null)); + ChatCompletionMessage.Role.ASSISTANT, null, null, toolCalls, null, audioOutput)); } else if (message.getMessageType() == MessageType.TOOL) { ToolResponseMessage toolMessage = (ToolResponseMessage) message; diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java index 0d5e6a4b3..89a18ce06 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java @@ -33,6 +33,7 @@ import org.springframework.ai.model.ModelOptionsUtils; import org.springframework.ai.model.function.FunctionCallback; import org.springframework.ai.model.function.FunctionCallingOptions; import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.StreamOptions; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ToolChoiceBuilder; import org.springframework.ai.openai.api.ResponseFormat; @@ -92,6 +93,27 @@ public class OpenAiChatOptions implements FunctionCallingOptions { * on the number of generated tokens across all of the choices. Keep n as 1 to minimize costs. */ private @JsonProperty("n") Integer n; + + /** + * Output types that you would like the model to generate for this request. + * Most models are capable of generating text, which is the default. + * The gpt-4o-audio-preview model can also be used to generate audio. + * To request that this model generate both text and audio responses, + * you can use: ["text", "audio"]. + * Note that the audio modality is only available for the gpt-4o-audio-preview model + * and is not supported for streaming completions. + */ + private @JsonProperty("modalities") List outputModalities; + + /** + * Audio parameters for the audio generation. Required when audio output is requested with + * modalities: ["audio"] + * Note: that the audio modality is only available for the gpt-4o-audio-preview model + * and is not supported for streaming completions. + + */ + private @JsonProperty("audio") AudioParameters outputAudio; + /** * Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they * appear in the text so far, increasing the model's likelihood to talk about new topics. @@ -206,6 +228,8 @@ public class OpenAiChatOptions implements FunctionCallingOptions { .withMaxTokens(fromOptions.getMaxTokens()) .withMaxCompletionTokens(fromOptions.getMaxCompletionTokens()) .withN(fromOptions.getN()) + .withOutputModalities(fromOptions.getOutputModalities()) + .withOutputAudio(fromOptions.getOutputAudio()) .withPresencePenalty(fromOptions.getPresencePenalty()) .withResponseFormat(fromOptions.getResponseFormat()) .withStreamUsage(fromOptions.getStreamUsage()) @@ -300,6 +324,22 @@ public class OpenAiChatOptions implements FunctionCallingOptions { this.n = n; } + public List getOutputModalities() { + return outputModalities; + } + + public void setOutputModalities(List modalities) { + this.outputModalities = modalities; + } + + public AudioParameters getOutputAudio() { + return outputAudio; + } + + public void setOutputAudio(AudioParameters audio) { + this.outputAudio = audio; + } + @Override public Double getPresencePenalty() { return this.presencePenalty; @@ -465,7 +505,7 @@ public class OpenAiChatOptions implements FunctionCallingOptions { this.maxTokens, this.maxCompletionTokens, this.n, this.presencePenalty, this.responseFormat, this.streamOptions, this.seed, this.stop, this.temperature, this.topP, this.tools, this.toolChoice, this.user, this.parallelToolCalls, this.functionCallbacks, this.functions, this.httpHeaders, - this.proxyToolCalls, this.toolContext); + this.proxyToolCalls, this.toolContext, this.outputModalities, this.outputAudio); } @Override @@ -493,7 +533,9 @@ public class OpenAiChatOptions implements FunctionCallingOptions { && Objects.equals(this.functions, other.functions) && Objects.equals(this.httpHeaders, other.httpHeaders) && Objects.equals(this.toolContext, other.toolContext) - && Objects.equals(this.proxyToolCalls, other.proxyToolCalls); + && Objects.equals(this.proxyToolCalls, other.proxyToolCalls) + && Objects.equals(this.outputModalities, other.outputModalities) + && Objects.equals(this.outputAudio, other.outputAudio); } @Override @@ -558,6 +600,16 @@ public class OpenAiChatOptions implements FunctionCallingOptions { return this; } + public Builder withOutputModalities(List modalities) { + this.options.outputModalities = modalities; + return this; + } + + public Builder withOutputAudio(AudioParameters audio) { + this.options.outputAudio = audio; + return this; + } + public Builder withPresencePenalty(Double presencePenalty) { this.options.presencePenalty = presencePenalty; return this; diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java index 1b1e712f5..2e0018b9b 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java @@ -839,10 +839,10 @@ public class OpenAiApi { * @param model ID of the model to use. * @param audio Parameters for audio output. Required when audio output is requested with outputModalities: ["audio"]. */ - public ChatCompletionRequest(List messages, String model, AudioParameters audio) { + public ChatCompletionRequest(List messages, String model, AudioParameters audio, boolean stream) { this(messages, model, null, null, null, null, null, null, null, null, null, List.of(OutputModality.AUDIO, OutputModality.TEXT), audio, null, null, - null, null, null, false, null, null, null, + null, null, null, stream, null, null, null, null, null, null, null); } @@ -938,34 +938,34 @@ public class OpenAiApi { * Specifies the voice type. */ public enum Voice { - @JsonProperty("alloy") - ALLOY, - @JsonProperty("echo") - ECHO, - @JsonProperty("fable") - FABLE, - @JsonProperty("onyx") - ONYX, - @JsonProperty("nova") - NOVA, - @JsonProperty("shimmer") - SHIMMER + /** Alloy voice */ + @JsonProperty("alloy") ALLOY, + /** Echo voice */ + @JsonProperty("echo") ECHO, + /** Fable voice */ + @JsonProperty("fable") FABLE, + /** Onyx voice */ + @JsonProperty("onyx") ONYX, + /** Nova voice */ + @JsonProperty("nova") NOVA, + /** Shimmer voice */ + @JsonProperty("shimmer") SHIMMER } /** * Specifies the output audio format. */ public enum AudioResponseFormat { - @JsonProperty("mp3") - MP3, - @JsonProperty("flac") - FLAC, - @JsonProperty("opus") - OPUS, - @JsonProperty("pcm16") - PCM16, - @JsonProperty("wav") - WAV + /** MP3 format */ + @JsonProperty("mp3") MP3, + /** FLAC format */ + @JsonProperty("flac") FLAC, + /** OPUS format */ + @JsonProperty("opus") OPUS, + /** PCM16 format */ + @JsonProperty("pcm16") PCM16, + /** WAV format */ + @JsonProperty("wav") WAV } } @@ -1119,10 +1119,10 @@ public class OpenAiApi { @JsonProperty("format") Format format) { public enum Format { - @JsonProperty("mp3") - MP3, - @JsonProperty("wav") - WAV + /** MP3 audio format */ + @JsonProperty("mp3") MP3, + /** WAV audio format */ + @JsonProperty("wav") WAV } // @formatter:on } diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java index 9a2bfaf41..a3c098f66 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java @@ -35,6 +35,7 @@ import org.springframework.core.io.ClassPathResource; import org.springframework.http.ResponseEntity; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** * @author Christian Tzolov @@ -105,7 +106,7 @@ public class OpenAiApiIT { ChatCompletionRequest.AudioParameters.Voice.NOVA, ChatCompletionRequest.AudioParameters.AudioResponseFormat.MP3); ChatCompletionRequest chatCompletionRequest = new ChatCompletionRequest(List.of(chatCompletionMessage), - OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue(), audioParameters); + OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue(), audioParameters, false); ResponseEntity response = this.openAiApi.chatCompletionEntity(chatCompletionRequest); assertThat(response).isNotNull(); @@ -119,4 +120,19 @@ public class OpenAiApiIT { .containsIgnoringCase("leviosa"); } + @Test + void streamOutputAudio() { + ChatCompletionMessage chatCompletionMessage = new ChatCompletionMessage( + "What is the magic spell to make objects fly?", Role.USER); + ChatCompletionRequest.AudioParameters audioParameters = new ChatCompletionRequest.AudioParameters( + ChatCompletionRequest.AudioParameters.Voice.NOVA, + ChatCompletionRequest.AudioParameters.AudioResponseFormat.MP3); + ChatCompletionRequest chatCompletionRequest = new ChatCompletionRequest(List.of(chatCompletionMessage), + OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue(), audioParameters, true); + + assertThatThrownBy(() -> this.openAiApi.chatCompletionStream(chatCompletionRequest).collectList().block()) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("400 Bad Request from POST https://api.openai.com/v1/chat/completions"); + } + } diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java index 07f54e67f..7611ae8ed 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java @@ -33,7 +33,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.ai.chat.prompt.ChatOptionsBuilder; import reactor.core.publisher.Flux; import org.springframework.ai.chat.client.ChatClient; @@ -42,6 +41,7 @@ import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.ChatOptionsBuilder; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.chat.prompt.PromptTemplate; import org.springframework.ai.chat.prompt.SystemPromptTemplate; @@ -53,6 +53,9 @@ import org.springframework.ai.model.function.FunctionCallback; import org.springframework.ai.openai.OpenAiChatOptions; import org.springframework.ai.openai.OpenAiTestConfiguration; import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters.AudioResponseFormat; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters.Voice; import org.springframework.ai.openai.api.tool.MockWeatherService; import org.springframework.ai.openai.testutils.AbstractIT; import org.springframework.beans.factory.annotation.Value; @@ -63,6 +66,7 @@ import org.springframework.core.io.Resource; import org.springframework.util.MimeTypeUtils; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @SpringBootTest(classes = OpenAiTestConfiguration.class) @EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+") @@ -434,6 +438,44 @@ public class OpenAiChatModelIT extends AbstractIT { assertThat(content).containsAnyOf("bowl", "basket", "fruit stand"); } + @ParameterizedTest(name = "{0} : {displayName} ") + @ValueSource(strings = { "gpt-4o-audio-preview" }) + void multiModalityOutputAudio(String modelName) throws IOException { + var userMessage = new UserMessage("Tell me joke about Spring Framework"); + + ChatResponse response = chatModel.call(new Prompt(List.of(userMessage), + OpenAiChatOptions.builder() + .withModel(modelName) + .withOutputModalities(List.of("text", "audio")) + .withOutputAudio(new AudioParameters(Voice.ALLOY, AudioResponseFormat.WAV)) + .build())); + + logger.info(response.getResult().getOutput().getContent()); + assertThat(response.getResult().getOutput().getContent()).isNotEmpty(); + + byte[] audio = response.getResult().getOutput().getMedia().get(0).getDataAsByteArray(); + assertThat(audio).isNotEmpty(); + // AudioPlayer.play(audio); + } + + @ParameterizedTest(name = "{0} : {displayName} ") + @ValueSource(strings = { "gpt-4o-audio-preview" }) + void streamingMultiModalityOutputAudio(String modelName) throws IOException { + // var audioResource = new ClassPathResource("speech1.mp3"); + var userMessage = new UserMessage("Tell me joke about Spring Framework"); + + assertThatThrownBy(() -> chatModel + .stream(new Prompt(List.of(userMessage), + OpenAiChatOptions.builder() + .withModel(modelName) + .withOutputModalities(List.of("text", "audio")) + .withOutputAudio(new AudioParameters(Voice.ALLOY, AudioResponseFormat.WAV)) + .build())) + .collectList() + .block()).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Audio parameters are not supported for streaming requests."); + } + @ParameterizedTest(name = "{0} : {displayName} ") @ValueSource(strings = { "gpt-4o-audio-preview" }) void multiModalityInputAudio(String modelName) { diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AssistantMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AssistantMessage.java index 9a974ec2d..7449b189a 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AssistantMessage.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AssistantMessage.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import org.springframework.ai.model.Media; +import org.springframework.ai.model.MediaContent; import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; @@ -33,10 +35,12 @@ import org.springframework.util.CollectionUtils; * @author Christian Tzolov * @since 1.0.0 */ -public class AssistantMessage extends AbstractMessage { +public class AssistantMessage extends AbstractMessage implements MediaContent { private final List toolCalls; + protected final List media; + public AssistantMessage(String content) { this(content, Map.of()); } @@ -46,9 +50,16 @@ public class AssistantMessage extends AbstractMessage { } public AssistantMessage(String content, Map properties, List toolCalls) { + this(content, properties, toolCalls, List.of()); + } + + public AssistantMessage(String content, Map properties, List toolCalls, + List media) { super(MessageType.ASSISTANT, content, properties); Assert.notNull(toolCalls, "Tool calls must not be null"); + Assert.notNull(media, "Media must not be null"); this.toolCalls = toolCalls; + this.media = media; } public List getToolCalls() { @@ -59,6 +70,11 @@ public class AssistantMessage extends AbstractMessage { return !CollectionUtils.isEmpty(this.toolCalls); } + @Override + public List getMedia() { + return this.media; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -70,12 +86,12 @@ public class AssistantMessage extends AbstractMessage { if (!super.equals(o)) { return false; } - return Objects.equals(this.toolCalls, that.toolCalls); + return Objects.equals(this.toolCalls, that.toolCalls) && Objects.equals(this.media, that.media); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), this.toolCalls); + return Objects.hash(super.hashCode(), this.toolCalls, this.media); } @Override diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java index 5a7e7db57..19f700b1f 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java @@ -64,10 +64,6 @@ public class UserMessage extends AbstractMessage implements MediaContent { this.media = new ArrayList<>(media); } - public List getMedia(String... dummy) { - return this.media; - } - @Override public String toString() { return "UserMessage{" + "content='" + getContent() + '\'' + ", properties=" + this.metadata + ", messageType=" diff --git a/spring-ai-core/src/main/java/org/springframework/ai/model/Media.java b/spring-ai-core/src/main/java/org/springframework/ai/model/Media.java index a994b0a6d..f04fde901 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/model/Media.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/model/Media.java @@ -34,19 +34,46 @@ import org.springframework.util.MimeType; */ public class Media { + public static final String MEDIA_NO_ID = "nope"; + + private final String id; + private final MimeType mimeType; private final Object data; + /** + * Create a new Media instance. + * @param mimeType the media MIME type + * @param url the URL for the media data + */ public Media(MimeType mimeType, URL url) { Assert.notNull(mimeType, "MimeType must not be null"); this.mimeType = mimeType; this.data = url.toString(); + this.id = MEDIA_NO_ID; } + /** + * Create a new Media instance. + * @param mimeType the media MIME type + * @param resource the media resource + */ public Media(MimeType mimeType, Resource resource) { + this(mimeType, resource, MEDIA_NO_ID); + } + + /** + * Create a new Media instance. + * @param mimeType the media MIME type + * @param resource the media resource + * @param id the media id + */ + public Media(MimeType mimeType, Resource resource, String id) { Assert.notNull(mimeType, "MimeType must not be null"); + Assert.notNull(id, "Id must not be null"); this.mimeType = mimeType; + this.id = id; try { this.data = resource.getContentAsByteArray(); } @@ -55,6 +82,10 @@ public class Media { } } + /** + * Get the media MIME type + * @return the media MIME type + */ public MimeType getMimeType() { return this.mimeType; } @@ -67,4 +98,25 @@ public class Media { return this.data; } + /** + * Get the media data as a byte array + * @return the media data as a byte array + */ + public byte[] getDataAsByteArray() { + if (this.data instanceof byte[]) { + return (byte[]) this.data; + } + else { + throw new IllegalStateException("Media data is not a byte[]"); + } + } + + /** + * Get the media id + * @return the media id + */ + public String getId() { + return this.id; + } + } diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/images/spring-ai-message-api.jpg b/spring-ai-docs/src/main/antora/modules/ROOT/images/spring-ai-message-api.jpg index 7534e5e90..08efdaaff 100644 Binary files a/spring-ai-docs/src/main/antora/modules/ROOT/images/spring-ai-message-api.jpg and b/spring-ai-docs/src/main/antora/modules/ROOT/images/spring-ai-message-api.jpg differ diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc index 97af23c31..82a699385 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc @@ -30,7 +30,8 @@ This table compares various Chat Models supported by Spring AI, detailing their | xref::api/chat/nvidia-chat.adoc[NVIDIA (OpenAI-proxy)] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] | xref::api/chat/oci-genai/cohere-chat.adoc[OCI GenAI/Cohere] | text ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] | xref::api/chat/ollama-chat.adoc[Ollama] | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] -| xref::api/chat/openai-chat.adoc[OpenAI] | text, image, audio ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] +| xref::api/chat/openai-chat.adoc[OpenAI] a| In: text, image, audio +Out: text, audio ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] | xref::api/chat/perplexity-chat.adoc[Perplexity (OpenAI-proxy)] | text ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] | xref::api/chat/qianfan-chat.adoc[QianFan] | text ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] | xref::api/chat/zhipuai-chat.adoc[ZhiPu AI] | text ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/openai-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/openai-chat.adoc index 7f285ccd0..53c342099 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/openai-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/openai-chat.adoc @@ -107,6 +107,11 @@ The prefix `spring.ai.openai.chat` is the property prefix that lets you configur | spring.ai.openai.chat.options.maxTokens | (Deprecated in favour of `maxCompletionTokens`) The maximum number of tokens to generate in the chat completion. The total length of input tokens and generated tokens is limited by the model's context length. | - | spring.ai.openai.chat.options.maxCompletionTokens | An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. | - | spring.ai.openai.chat.options.n | How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as 1 to minimize costs. | 1 +| spring.ai.openai.chat.options.output-modalities | Output types that you would like the model to generate for this request. Most models are capable of generating text, which is the default. +The `gpt-4o-audio-preview` model can also be used to generate audio. To request that this model generate both text and audio responses, +you can use: `text`, `audio`. Not supported for streaming. | - +| spring.ai.openai.chat.options.output-audio | Audio parameters for the audio generation. Required when audio output is requested with `output-modalities`: `audio`. +Requires the `gpt-4o-audio-preview` model and is is not supported for streaming completions. | - | spring.ai.openai.chat.options.presencePenalty | Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. | - | spring.ai.openai.chat.options.responseFormat.type | Compatible with `GPT-4o`, `GPT-4o mini`, `GPT-4 Turbo` and all `GPT-3.5 Turbo` models newer than `gpt-3.5-turbo-1106`. The `JSON_OBJECT` type enables JSON mode, which guarantees the message the model generates is valid JSON. The `JSON_SCHEMA` type enables link:https://platform.openai.com/docs/guides/structured-outputs[Structured Outputs] which guarantees the model will match your supplied JSON schema. The JSON_SCHEMA type requires setting the `responseFormat.schema` property as well. | - @@ -221,7 +226,7 @@ view of the fruit inside. === Audio -OpenAI models that offer audio multimodal support include `gpt-4o-audio-preview`. +OpenAI models that offer input audio multimodal support include `gpt-4o-audio-preview`. Refer to the link:https://platform.openai.com/docs/guides/audio[Audio] guide for more information. The OpenAI link:https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages[User Message API] can incorporate a list of base64-encoded audio files with the message. @@ -244,6 +249,37 @@ ChatResponse response = chatModel.call(new Prompt(List.of(userMessage), TIP: You can pass multiple audio files as well. +=== Output Audio + +OpenAI models that offer input audio multimodal support include `gpt-4o-audio-preview`. +Refer to the link:https://platform.openai.com/docs/guides/audio[Audio] guide for more information. + +The OpenAI link:https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages[Assystant Message API] can contain a list of base64-encoded audio files with the message. +Spring AI’s link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java[Message] interface facilitates multimodal AI models by introducing the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java[Media] type. +This type encompasses data and details regarding media attachments in messages, utilizing Spring’s `org.springframework.util.MimeType` and a `org.springframework.core.io.Resource` for the raw media data. +Currently, OpenAI support only the following audio types: `audio/mp3` and `audio/wav`. + +Below is a code example, illustrating the response of user text along with an audio byte array, using the `gpt-4o-audio-preview` model: + +[source,java] +---- +var userMessage = new UserMessage("Tell me joke about Spring Framework"); + +ChatResponse response = chatModel.call(new Prompt(List.of(userMessage), + OpenAiChatOptions.builder() + .withModel(OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW) + .withOutputModalities(List.of("text", "audio")) + .withOutputAudio(new AudioParameters(Voice.ALLOY, AudioResponseFormat.WAV)) + .build())); + +String text = response.getResult().getOutput().getContent(); // audio transcript + +byte[] waveAudio = response.getResult().getOutput().getMedia().get(0).getDataAsByteArray(); // audio data +---- + +You have to specify an `audio` modality in the `OpenAiChatOptions` to generate audio output. +The `AudioParameters` class provides the voice and audio format for the audio output. + == Structured Outputs OpenAI provides custom https://platform.openai.com/docs/guides/structured-outputs[Structured Outputs] APIs that ensure your model generates responses conforming strictly to your provided `JSON Schema`. diff --git a/spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 392f16117..3a94062ad 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -10,6 +10,11 @@ "name": "spring.ai.mistralai.chat.options.tool-choice", "type": "org.springframework.ai.mistralai.api.MistralAiApi$ChatCompletionRequest$ToolChoice", "sourceType": "org.springframework.ai.mistralai.MistralAiChatOptions" + }, + { + "name": "spring.ai.openai.chat.output-audio", + "type": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters", + "sourceType": "org.springframework.ai.openai.OpenAiChatOptions" } ], "properties": [ @@ -22,6 +27,16 @@ "name": "spring.ai.azure.openai.chat.options.enhancements.ocr", "type": "com.azure.ai.openai.models.AzureChatOCREnhancementConfiguration", "sourceType": "com.azure.ai.openai.models.AzureChatEnhancementConfiguration" + }, + { + "name": "spring.ai.openai.chat.output-audio.voice", + "type": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters$Voice", + "sourceType": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters" + }, + { + "name": "spring.ai.openai.chat.output-audio.format", + "type": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters$AudioResponseFormat", + "sourceType": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters" } ], "hints": [] diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java index 184848e30..c72052920 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java @@ -38,6 +38,7 @@ import org.springframework.ai.openai.OpenAiAudioTranscriptionModel; import org.springframework.ai.openai.OpenAiChatModel; import org.springframework.ai.openai.OpenAiEmbeddingModel; import org.springframework.ai.openai.OpenAiImageModel; +import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.core.io.ClassPathResource; @@ -55,7 +56,7 @@ public class OpenAiAutoConfigurationIT { .withConfiguration(AutoConfigurations.of(OpenAiAutoConfiguration.class)); @Test - void generate() { + void chatCall() { this.contextRunner.run(context -> { OpenAiChatModel chatModel = context.getBean(OpenAiChatModel.class); String response = chatModel.call("Hello"); @@ -64,6 +65,25 @@ public class OpenAiAutoConfigurationIT { }); } + @Test + void chatCallAudioResponse() { + this.contextRunner + .withPropertyValues( + "spring.ai.openai.chat.options.model=" + OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue(), + "spring.ai.openai.chat.options.output-modalities=text,audio", + "spring.ai.openai.chat.options.output-audio.voice=ALLOY", + "spring.ai.openai.chat.options.output-audio.format=WAV") + .run(context -> { + OpenAiChatModel chatModel = context.getBean(OpenAiChatModel.class); + + ChatResponse response = chatModel + .call(new Prompt(new UserMessage("Tell me joke about Spring Framework"))); + assertThat(response).isNotNull(); + logger.info("Response: " + response); + // AudioPlayer.play(response.getResult().getOutput().getMedia().get(0).getDataAsByteArray()); + }); + } + @Test void transcribe() { this.contextRunner.run(context -> { diff --git a/spring-ai-test/src/main/java/org/springframework/ai/utils/AudioPlayer.java b/spring-ai-test/src/main/java/org/springframework/ai/utils/AudioPlayer.java new file mode 100644 index 000000000..3c744e205 --- /dev/null +++ b/spring-ai-test/src/main/java/org/springframework/ai/utils/AudioPlayer.java @@ -0,0 +1,65 @@ +/* +* Copyright 2024 - 2024 the original author or authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.springframework.ai.utils; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.InputStream; + +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.Clip; + +/** + * @author Christian Tzolov + * @since 1.0.0 + */ +public class AudioPlayer { + + public static void main(String[] args) throws Exception { + + play(new BufferedInputStream(new FileInputStream( + "/Users/christiantzolov/Dev/projects/spring-ai/models/spring-ai-openai/output.wav"))); + } + + public static void play(byte[] data) { + play(new BufferedInputStream(new ByteArrayInputStream(data))); + } + + public static void play(InputStream data) { + + try { + try (AudioInputStream audio = AudioSystem.getAudioInputStream(data); Clip clip = AudioSystem.getClip()) { + clip.open(audio); + clip.start(); + // wait to start + while (!clip.isRunning()) { + Thread.sleep(100); + } + // wait to finish + while (clip.isRunning()) { + Thread.sleep(3000); + } + } + } + catch (Exception e) { + throw new RuntimeException(e); + } + + } + +}