feat(openai) - Support for audio output in OpenAI chat model

- Introduced new options for audio output modalities in ChatCompletionRequest
- Added AudioParameters configuration for voice and audio format selection
- Enhanced OpenAiChatModel to handle audio generation and embedding
- Updated AssistantMessage and Media classes to support audio media
- Added integration tests for audio output functionality
- Implemented support for text and audio multi-modal responses
- Updated Spring AI's chat model comparison table to clarify OpenAI's input/output modalities
- Added new configuration properties for audio output:
  * spring.ai.openai.chat.options.output-modalities
  * spring.ai.openai.chat.options.output-audio
- Extended documentation to explain audio output generation with the gpt-4o-audio-preview model
- Updated Spring Boot configuration metadata to support new audio-related properties
- Included auto-configuration integration test for chat model with audio response generation

Resolves #1841

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java

@@ -64,6 +64,7 @@ import org.springframework.ai.openai.api.OpenAiApi;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletion;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletion.Choice;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage;
+import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.AudioOutput;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.ChatCompletionFunction;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.MediaContent;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.ToolCall;
@@ -72,6 +73,8 @@ import org.springframework.ai.openai.api.common.OpenAiApiConstants;
 import org.springframework.ai.openai.metadata.OpenAiUsage;
 import org.springframework.ai.openai.metadata.support.OpenAiResponseHeaderExtractor;
 import org.springframework.ai.retry.RetryUtils;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.core.io.Resource;
 import org.springframework.http.ResponseEntity;
 import org.springframework.retry.support.RetryTemplate;
 import org.springframework.util.Assert;
@@ -251,7 +254,7 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
 							"finishReason", choice.finishReason() != null ? choice.finishReason().name() : "",
 							"refusal", StringUtils.hasText(choice.message().refusal()) ? choice.message().refusal() : "");
 					// @formatter:on
-					return buildGeneration(choice, metadata);
+					return buildGeneration(choice, metadata, request);
 				}).toList();
 
 				// Non function calling.
@@ -282,6 +285,17 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
 		return Flux.deferContextual(contextView -> {
 			ChatCompletionRequest request = createRequest(prompt, true);
 
+			if (request.outputModalities() != null) {
+				if (request.outputModalities().stream().anyMatch(m -> m.equals("audio"))) {
+					logger.warn("Audio output is not supported for streaming requests. Removing audio output.");
+					throw new IllegalArgumentException("Audio output is not supported for streaming requests.");
+				}
+			}
+			if (request.audioParameters() != null) {
+				logger.warn("Audio parameters are not supported for streaming requests. Removing audio parameters.");
+				throw new IllegalArgumentException("Audio parameters are not supported for streaming requests.");
+			}
+
 			Flux<OpenAiApi.ChatCompletionChunk> completionChunks = this.openAiApi.chatCompletionStream(request,
 					getAdditionalHttpHeaders(prompt));
 
@@ -320,7 +334,7 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
 									"finishReason", choice.finishReason() != null ? choice.finishReason().name() : "",
 									"refusal", StringUtils.hasText(choice.message().refusal()) ? choice.message().refusal() : "");
 
-							return buildGeneration(choice, metadata);
+							return buildGeneration(choice, metadata, request);
 						}).toList();
 						// @formatter:on
 
@@ -367,7 +381,7 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
 				headers.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> List.of(e.getValue()))));
 	}
 
-	private Generation buildGeneration(Choice choice, Map<String, Object> metadata) {
+	private Generation buildGeneration(Choice choice, Map<String, Object> metadata, ChatCompletionRequest request) {
 		List<AssistantMessage.ToolCall> toolCalls = choice.message().toolCalls() == null ? List.of()
 				: choice.message()
 					.toolCalls()
@@ -376,10 +390,26 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
 							toolCall.function().name(), toolCall.function().arguments()))
 					.toList();
 
-		var assistantMessage = new AssistantMessage(choice.message().content(), metadata, toolCalls);
 		String finishReason = (choice.finishReason() != null ? choice.finishReason().name() : "");
-		var generationMetadata = ChatGenerationMetadata.builder().finishReason(finishReason).build();
-		return new Generation(assistantMessage, generationMetadata);
+		var generationMetadataBuilder = ChatGenerationMetadata.builder().finishReason(finishReason);
+
+		List<Media> media = new ArrayList<>();
+		String textContent = choice.message().content();
+		var audioOutput = choice.message().audioOutput();
+		if (audioOutput != null) {
+			String mimeType = String.format("audio/%s", request.audioParameters().format().name().toLowerCase());
+			byte[] audioData = Base64.getDecoder().decode(audioOutput.data());
+			Resource resource = new ByteArrayResource(audioData);
+			media.add(new Media(MimeTypeUtils.parseMimeType(mimeType), resource, audioOutput.id()));
+			if (!StringUtils.hasText(textContent)) {
+				textContent = audioOutput.transcript();
+			}
+			generationMetadataBuilder.metadata("audioId", audioOutput.id());
+			generationMetadataBuilder.metadata("audioExpiresAt", audioOutput.expiresAt());
+		}
+
+		var assistantMessage = new AssistantMessage(textContent, metadata, toolCalls, media);
+		return new Generation(assistantMessage, generationMetadataBuilder.build());
 	}
 
 	private ChatResponseMetadata from(OpenAiApi.ChatCompletion result, RateLimit rateLimit) {
@@ -443,8 +473,15 @@ public class OpenAiChatModel extends AbstractToolCallSupport implements ChatMode
 						return new ToolCall(toolCall.id(), toolCall.type(), function);
 					}).toList();
 				}
+				AudioOutput audioOutput = null;
+				if (!CollectionUtils.isEmpty(assistantMessage.getMedia())) {
+					Assert.isTrue(assistantMessage.getMedia().size() == 1,
+							"Only one media content is supported for assistant messages");
+					audioOutput = new AudioOutput(assistantMessage.getMedia().get(0).getId(), null, null, null);
+
+				}
 				return List.of(new ChatCompletionMessage(assistantMessage.getContent(),
-						ChatCompletionMessage.Role.ASSISTANT, null, null, toolCalls, null, null));
+						ChatCompletionMessage.Role.ASSISTANT, null, null, toolCalls, null, audioOutput));
 			}
 			else if (message.getMessageType() == MessageType.TOOL) {
 				ToolResponseMessage toolMessage = (ToolResponseMessage) message;

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java

@@ -33,6 +33,7 @@ import org.springframework.ai.model.ModelOptionsUtils;
 import org.springframework.ai.model.function.FunctionCallback;
 import org.springframework.ai.model.function.FunctionCallingOptions;
 import org.springframework.ai.openai.api.OpenAiApi;
+import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.StreamOptions;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ToolChoiceBuilder;
 import org.springframework.ai.openai.api.ResponseFormat;
@@ -92,6 +93,27 @@ public class OpenAiChatOptions implements FunctionCallingOptions {
 	 * on the number of generated tokens across all of the choices. Keep n as 1 to minimize costs.
 	 */
 	private @JsonProperty("n") Integer n;
+
+	/**
+	 * Output types that you would like the model to generate for this request.
+	 * Most models are capable of generating text, which is the default.
+	 * The gpt-4o-audio-preview model can also be used to generate audio.
+	 * To request that this model generate both text and audio responses,
+	 * you can use: ["text", "audio"].
+	 * Note that the audio modality is only available for the gpt-4o-audio-preview model
+	 * and is not supported for streaming completions.
+	 */
+	private @JsonProperty("modalities") List<String> outputModalities;
+
+	/**
+	 * Audio parameters for the audio generation. Required when audio output is requested with
+	 * modalities: ["audio"]
+	 * Note: that the audio modality is only available for the gpt-4o-audio-preview model
+	 * and is not supported for streaming completions.
+
+	 */
+	private @JsonProperty("audio") AudioParameters outputAudio;
+
 	/**
 	 * Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they
 	 * appear in the text so far, increasing the model's likelihood to talk about new topics.
@@ -206,6 +228,8 @@ public class OpenAiChatOptions implements FunctionCallingOptions {
 			.withMaxTokens(fromOptions.getMaxTokens())
 			.withMaxCompletionTokens(fromOptions.getMaxCompletionTokens())
 			.withN(fromOptions.getN())
+			.withOutputModalities(fromOptions.getOutputModalities())
+			.withOutputAudio(fromOptions.getOutputAudio())
 			.withPresencePenalty(fromOptions.getPresencePenalty())
 			.withResponseFormat(fromOptions.getResponseFormat())
 			.withStreamUsage(fromOptions.getStreamUsage())
@@ -300,6 +324,22 @@ public class OpenAiChatOptions implements FunctionCallingOptions {
 		this.n = n;
 	}
 
+	public List<String> getOutputModalities() {
+		return outputModalities;
+	}
+
+	public void setOutputModalities(List<String> modalities) {
+		this.outputModalities = modalities;
+	}
+
+	public AudioParameters getOutputAudio() {
+		return outputAudio;
+	}
+
+	public void setOutputAudio(AudioParameters audio) {
+		this.outputAudio = audio;
+	}
+
 	@Override
 	public Double getPresencePenalty() {
 		return this.presencePenalty;
@@ -465,7 +505,7 @@ public class OpenAiChatOptions implements FunctionCallingOptions {
 				this.maxTokens, this.maxCompletionTokens, this.n, this.presencePenalty, this.responseFormat,
 				this.streamOptions, this.seed, this.stop, this.temperature, this.topP, this.tools, this.toolChoice,
 				this.user, this.parallelToolCalls, this.functionCallbacks, this.functions, this.httpHeaders,
-				this.proxyToolCalls, this.toolContext);
+				this.proxyToolCalls, this.toolContext, this.outputModalities, this.outputAudio);
 	}
 
 	@Override
@@ -493,7 +533,9 @@ public class OpenAiChatOptions implements FunctionCallingOptions {
 				&& Objects.equals(this.functions, other.functions)
 				&& Objects.equals(this.httpHeaders, other.httpHeaders)
 				&& Objects.equals(this.toolContext, other.toolContext)
-				&& Objects.equals(this.proxyToolCalls, other.proxyToolCalls);
+				&& Objects.equals(this.proxyToolCalls, other.proxyToolCalls)
+				&& Objects.equals(this.outputModalities, other.outputModalities)
+				&& Objects.equals(this.outputAudio, other.outputAudio);
 	}
 
 	@Override
@@ -558,6 +600,16 @@ public class OpenAiChatOptions implements FunctionCallingOptions {
 			return this;
 		}
 
+		public Builder withOutputModalities(List<String> modalities) {
+			this.options.outputModalities = modalities;
+			return this;
+		}
+
+		public Builder withOutputAudio(AudioParameters audio) {
+			this.options.outputAudio = audio;
+			return this;
+		}
+
 		public Builder withPresencePenalty(Double presencePenalty) {
 			this.options.presencePenalty = presencePenalty;
 			return this;

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java

@@ -839,10 +839,10 @@ public class OpenAiApi {
 		 * @param model ID of the model to use.
 		 * @param audio Parameters for audio output. Required when audio output is requested with outputModalities: ["audio"].
 		 */
-		public ChatCompletionRequest(List<ChatCompletionMessage> messages, String model, AudioParameters audio) {
+		public ChatCompletionRequest(List<ChatCompletionMessage> messages, String model, AudioParameters audio, boolean stream) {
 			this(messages, model, null, null, null, null, null, null,
 					null, null, null, List.of(OutputModality.AUDIO, OutputModality.TEXT), audio, null, null,
-					null, null, null, false, null, null, null,
+					null, null, null, stream, null, null, null,
 					null, null, null, null);
 		}
 
@@ -938,34 +938,34 @@ public class OpenAiApi {
 			 * Specifies the voice type.
 			 */
 			public enum Voice {
-				@JsonProperty("alloy")
-				ALLOY,
-				@JsonProperty("echo")
-				ECHO,
-				@JsonProperty("fable")
-				FABLE,
-				@JsonProperty("onyx")
-				ONYX,
-				@JsonProperty("nova")
-				NOVA,
-				@JsonProperty("shimmer")
-				SHIMMER
+				/** Alloy voice */
+				@JsonProperty("alloy") ALLOY,
+				/** Echo voice */
+				@JsonProperty("echo") ECHO,
+				/** Fable voice */
+				@JsonProperty("fable") FABLE,
+				/** Onyx voice */
+				@JsonProperty("onyx") ONYX,
+				/** Nova voice */
+				@JsonProperty("nova") NOVA,
+				/** Shimmer voice */
+				@JsonProperty("shimmer") SHIMMER
 			}
 
 			/**
 			 * Specifies the output audio format.
 			 */
 			public enum AudioResponseFormat {
-				@JsonProperty("mp3")
-				MP3,
-				@JsonProperty("flac")
-				FLAC,
-				@JsonProperty("opus")
-				OPUS,
-				@JsonProperty("pcm16")
-				PCM16,
-				@JsonProperty("wav")
-				WAV
+				/** MP3 format */
+				@JsonProperty("mp3") MP3,
+				/** FLAC format */
+				@JsonProperty("flac") FLAC,
+				/** OPUS format */	
+				@JsonProperty("opus") OPUS,
+				/** PCM16 format */
+				@JsonProperty("pcm16") PCM16,
+				/** WAV format */
+				@JsonProperty("wav") WAV
 			}
 		}
 
@@ -1119,10 +1119,10 @@ public class OpenAiApi {
 				@JsonProperty("format") Format format) {
 
 				public enum Format {
-					@JsonProperty("mp3")
-					MP3,
-					@JsonProperty("wav")
-					WAV
+					/** MP3 audio format */
+					@JsonProperty("mp3") MP3,
+					/** WAV audio format */
+					@JsonProperty("wav") WAV
 				} // @formatter:on
 			}
 

models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java

@@ -35,6 +35,7 @@ import org.springframework.core.io.ClassPathResource;
 import org.springframework.http.ResponseEntity;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 /**
  * @author Christian Tzolov
@@ -105,7 +106,7 @@ public class OpenAiApiIT {
 				ChatCompletionRequest.AudioParameters.Voice.NOVA,
 				ChatCompletionRequest.AudioParameters.AudioResponseFormat.MP3);
 		ChatCompletionRequest chatCompletionRequest = new ChatCompletionRequest(List.of(chatCompletionMessage),
-				OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue(), audioParameters);
+				OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue(), audioParameters, false);
 		ResponseEntity<ChatCompletion> response = this.openAiApi.chatCompletionEntity(chatCompletionRequest);
 
 		assertThat(response).isNotNull();
@@ -119,4 +120,19 @@ public class OpenAiApiIT {
 			.containsIgnoringCase("leviosa");
 	}
 
+	@Test
+	void streamOutputAudio() {
+		ChatCompletionMessage chatCompletionMessage = new ChatCompletionMessage(
+				"What is the magic spell to make objects fly?", Role.USER);
+		ChatCompletionRequest.AudioParameters audioParameters = new ChatCompletionRequest.AudioParameters(
+				ChatCompletionRequest.AudioParameters.Voice.NOVA,
+				ChatCompletionRequest.AudioParameters.AudioResponseFormat.MP3);
+		ChatCompletionRequest chatCompletionRequest = new ChatCompletionRequest(List.of(chatCompletionMessage),
+				OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue(), audioParameters, true);
+
+		assertThatThrownBy(() -> this.openAiApi.chatCompletionStream(chatCompletionRequest).collectList().block())
+			.isInstanceOf(RuntimeException.class)
+			.hasMessageContaining("400 Bad Request from POST https://api.openai.com/v1/chat/completions");
+	}
+
 }

models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java

@@ -33,7 +33,6 @@ import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.ai.chat.prompt.ChatOptionsBuilder;
 import reactor.core.publisher.Flux;
 
 import org.springframework.ai.chat.client.ChatClient;
@@ -42,6 +41,7 @@ import org.springframework.ai.chat.messages.Message;
 import org.springframework.ai.chat.messages.UserMessage;
 import org.springframework.ai.chat.model.ChatResponse;
 import org.springframework.ai.chat.model.Generation;
+import org.springframework.ai.chat.prompt.ChatOptionsBuilder;
 import org.springframework.ai.chat.prompt.Prompt;
 import org.springframework.ai.chat.prompt.PromptTemplate;
 import org.springframework.ai.chat.prompt.SystemPromptTemplate;
@@ -53,6 +53,9 @@ import org.springframework.ai.model.function.FunctionCallback;
 import org.springframework.ai.openai.OpenAiChatOptions;
 import org.springframework.ai.openai.OpenAiTestConfiguration;
 import org.springframework.ai.openai.api.OpenAiApi;
+import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters;
+import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters.AudioResponseFormat;
+import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters.Voice;
 import org.springframework.ai.openai.api.tool.MockWeatherService;
 import org.springframework.ai.openai.testutils.AbstractIT;
 import org.springframework.beans.factory.annotation.Value;
@@ -63,6 +66,7 @@ import org.springframework.core.io.Resource;
 import org.springframework.util.MimeTypeUtils;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 @SpringBootTest(classes = OpenAiTestConfiguration.class)
 @EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+")
@@ -434,6 +438,44 @@ public class OpenAiChatModelIT extends AbstractIT {
 		assertThat(content).containsAnyOf("bowl", "basket", "fruit stand");
 	}
 
+	@ParameterizedTest(name = "{0} : {displayName} ")
+	@ValueSource(strings = { "gpt-4o-audio-preview" })
+	void multiModalityOutputAudio(String modelName) throws IOException {
+		var userMessage = new UserMessage("Tell me joke about Spring Framework");
+
+		ChatResponse response = chatModel.call(new Prompt(List.of(userMessage),
+				OpenAiChatOptions.builder()
+					.withModel(modelName)
+					.withOutputModalities(List.of("text", "audio"))
+					.withOutputAudio(new AudioParameters(Voice.ALLOY, AudioResponseFormat.WAV))
+					.build()));
+
+		logger.info(response.getResult().getOutput().getContent());
+		assertThat(response.getResult().getOutput().getContent()).isNotEmpty();
+
+		byte[] audio = response.getResult().getOutput().getMedia().get(0).getDataAsByteArray();
+		assertThat(audio).isNotEmpty();
+		// AudioPlayer.play(audio);
+	}
+
+	@ParameterizedTest(name = "{0} : {displayName} ")
+	@ValueSource(strings = { "gpt-4o-audio-preview" })
+	void streamingMultiModalityOutputAudio(String modelName) throws IOException {
+		// var audioResource = new ClassPathResource("speech1.mp3");
+		var userMessage = new UserMessage("Tell me joke about Spring Framework");
+
+		assertThatThrownBy(() -> chatModel
+			.stream(new Prompt(List.of(userMessage),
+					OpenAiChatOptions.builder()
+						.withModel(modelName)
+						.withOutputModalities(List.of("text", "audio"))
+						.withOutputAudio(new AudioParameters(Voice.ALLOY, AudioResponseFormat.WAV))
+						.build()))
+			.collectList()
+			.block()).isInstanceOf(IllegalArgumentException.class)
+			.hasMessageContaining("Audio parameters are not supported for streaming requests.");
+	}
+
 	@ParameterizedTest(name = "{0} : {displayName} ")
 	@ValueSource(strings = { "gpt-4o-audio-preview" })
 	void multiModalityInputAudio(String modelName) {

spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AssistantMessage.java

@@ -20,6 +20,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
+import org.springframework.ai.model.Media;
+import org.springframework.ai.model.MediaContent;
 import org.springframework.util.Assert;
 import org.springframework.util.CollectionUtils;
 
@@ -33,10 +35,12 @@ import org.springframework.util.CollectionUtils;
  * @author Christian Tzolov
  * @since 1.0.0
  */
-public class AssistantMessage extends AbstractMessage {
+public class AssistantMessage extends AbstractMessage implements MediaContent {
 
 	private final List<ToolCall> toolCalls;
 
+	protected final List<Media> media;
+
 	public AssistantMessage(String content) {
 		this(content, Map.of());
 	}
@@ -46,9 +50,16 @@ public class AssistantMessage extends AbstractMessage {
 	}
 
 	public AssistantMessage(String content, Map<String, Object> properties, List<ToolCall> toolCalls) {
+		this(content, properties, toolCalls, List.of());
+	}
+
+	public AssistantMessage(String content, Map<String, Object> properties, List<ToolCall> toolCalls,
+			List<Media> media) {
 		super(MessageType.ASSISTANT, content, properties);
 		Assert.notNull(toolCalls, "Tool calls must not be null");
+		Assert.notNull(media, "Media must not be null");
 		this.toolCalls = toolCalls;
+		this.media = media;
 	}
 
 	public List<ToolCall> getToolCalls() {
@@ -59,6 +70,11 @@ public class AssistantMessage extends AbstractMessage {
 		return !CollectionUtils.isEmpty(this.toolCalls);
 	}
 
+	@Override
+	public List<Media> getMedia() {
+		return this.media;
+	}
+
 	@Override
 	public boolean equals(Object o) {
 		if (this == o) {
@@ -70,12 +86,12 @@ public class AssistantMessage extends AbstractMessage {
 		if (!super.equals(o)) {
 			return false;
 		}
-		return Objects.equals(this.toolCalls, that.toolCalls);
+		return Objects.equals(this.toolCalls, that.toolCalls) && Objects.equals(this.media, that.media);
 	}
 
 	@Override
 	public int hashCode() {
-		return Objects.hash(super.hashCode(), this.toolCalls);
+		return Objects.hash(super.hashCode(), this.toolCalls, this.media);
 	}
 
 	@Override

spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java

@@ -64,10 +64,6 @@ public class UserMessage extends AbstractMessage implements MediaContent {
 		this.media = new ArrayList<>(media);
 	}
 
-	public List<Media> getMedia(String... dummy) {
-		return this.media;
-	}
-
 	@Override
 	public String toString() {
 		return "UserMessage{" + "content='" + getContent() + '\'' + ", properties=" + this.metadata + ", messageType="

spring-ai-core/src/main/java/org/springframework/ai/model/Media.java

@@ -34,19 +34,46 @@ import org.springframework.util.MimeType;
  */
 public class Media {
 
+	public static final String MEDIA_NO_ID = "nope";
+
+	private final String id;
+
 	private final MimeType mimeType;
 
 	private final Object data;
 
+	/**
+	 * Create a new Media instance.
+	 * @param mimeType the media MIME type
+	 * @param url the URL for the media data
+	 */
 	public Media(MimeType mimeType, URL url) {
 		Assert.notNull(mimeType, "MimeType must not be null");
 		this.mimeType = mimeType;
 		this.data = url.toString();
+		this.id = MEDIA_NO_ID;
 	}
 
+	/**
+	 * Create a new Media instance.
+	 * @param mimeType the media MIME type
+	 * @param resource the media resource
+	 */
 	public Media(MimeType mimeType, Resource resource) {
+		this(mimeType, resource, MEDIA_NO_ID);
+	}
+
+	/**
+	 * Create a new Media instance.
+	 * @param mimeType the media MIME type
+	 * @param resource the media resource
+	 * @param id the media id
+	 */
+	public Media(MimeType mimeType, Resource resource, String id) {
 		Assert.notNull(mimeType, "MimeType must not be null");
+		Assert.notNull(id, "Id must not be null");
 		this.mimeType = mimeType;
+		this.id = id;
 		try {
 			this.data = resource.getContentAsByteArray();
 		}
@@ -55,6 +82,10 @@ public class Media {
 		}
 	}
 
+	/**
+	 * Get the media MIME type
+	 * @return the media MIME type
+	 */
 	public MimeType getMimeType() {
 		return this.mimeType;
 	}
@@ -67,4 +98,25 @@ public class Media {
 		return this.data;
 	}
 
+	/**
+	 * Get the media data as a byte array
+	 * @return the media data as a byte array
+	 */
+	public byte[] getDataAsByteArray() {
+		if (this.data instanceof byte[]) {
+			return (byte[]) this.data;
+		}
+		else {
+			throw new IllegalStateException("Media data is not a byte[]");
+		}
+	}
+
+	/**
+	 * Get the media id
+	 * @return the media id
+	 */
+	public String getId() {
+		return this.id;
+	}
+
 }

spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/comparison.adoc

@@ -30,7 +30,8 @@ This table compares various Chat Models supported by Spring AI, detailing their
 | xref::api/chat/nvidia-chat.adoc[NVIDIA (OpenAI-proxy)]  | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
 | xref::api/chat/oci-genai/cohere-chat.adoc[OCI GenAI/Cohere] | text ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12]
 | xref::api/chat/ollama-chat.adoc[Ollama]  | text, image ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16]
-| xref::api/chat/openai-chat.adoc[OpenAI]  | text, image, audio ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
+| xref::api/chat/openai-chat.adoc[OpenAI]  a| In: text, image, audio
+Out: text, audio ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
 | xref::api/chat/perplexity-chat.adoc[Perplexity (OpenAI-proxy)]  | text ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16]
 | xref::api/chat/qianfan-chat.adoc[QianFan]  | text ^a| image::no.svg[width=12] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12]
 | xref::api/chat/zhipuai-chat.adoc[ZhiPu AI]  | text ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::yes.svg[width=16] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12] ^a| image::no.svg[width=12]

spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/openai-chat.adoc

@@ -107,6 +107,11 @@ The prefix `spring.ai.openai.chat` is the property prefix that lets you configur
 | spring.ai.openai.chat.options.maxTokens | (Deprecated in favour of `maxCompletionTokens`) The maximum number of tokens to generate in the chat completion. The total length of input tokens and generated tokens is limited by the model's context length. | -
 | spring.ai.openai.chat.options.maxCompletionTokens | An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. | -
 | spring.ai.openai.chat.options.n | How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as 1 to minimize costs. | 1
+| spring.ai.openai.chat.options.output-modalities | Output types that you would like the model to generate for this request. Most models are capable of generating text, which is the default.
+The `gpt-4o-audio-preview` model can also be used to generate audio. To request that this model generate both text and audio responses,
+you can use: `text`, `audio`. Not supported for streaming. | -
+| spring.ai.openai.chat.options.output-audio | Audio parameters for the audio generation. Required when audio output is requested with `output-modalities`: `audio`.
+Requires the `gpt-4o-audio-preview` model and is is not supported for streaming completions. | -
 | spring.ai.openai.chat.options.presencePenalty | Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. | -
 | spring.ai.openai.chat.options.responseFormat.type | Compatible with `GPT-4o`, `GPT-4o mini`, `GPT-4 Turbo` and all `GPT-3.5 Turbo` models newer than `gpt-3.5-turbo-1106`. The `JSON_OBJECT` type enables JSON mode, which guarantees the message the model generates is valid JSON.
 The `JSON_SCHEMA` type enables link:https://platform.openai.com/docs/guides/structured-outputs[Structured Outputs] which guarantees the model will match your supplied JSON schema. The JSON_SCHEMA type requires setting the `responseFormat.schema` property as well. | -
@@ -221,7 +226,7 @@ view of the fruit inside.
 
 === Audio
 
-OpenAI models that offer audio multimodal support include `gpt-4o-audio-preview`.
+OpenAI models that offer input audio multimodal support include `gpt-4o-audio-preview`.
 Refer to the link:https://platform.openai.com/docs/guides/audio[Audio] guide for more information.
 
 The OpenAI link:https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages[User Message API] can incorporate a list of base64-encoded audio files with the message.
@@ -244,6 +249,37 @@ ChatResponse response = chatModel.call(new Prompt(List.of(userMessage),
 
 TIP: You can pass multiple audio files as well.
 
+=== Output Audio
+
+OpenAI models that offer input audio multimodal support include `gpt-4o-audio-preview`.
+Refer to the link:https://platform.openai.com/docs/guides/audio[Audio] guide for more information.
+
+The OpenAI link:https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages[Assystant Message API] can contain a list of base64-encoded audio files with the message.
+Spring AI’s link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java[Message] interface facilitates multimodal AI models by introducing the link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java[Media] type.
+This type encompasses data and details regarding media attachments in messages, utilizing Spring’s `org.springframework.util.MimeType` and a `org.springframework.core.io.Resource` for the raw media data.
+Currently, OpenAI support only the following audio types: `audio/mp3` and `audio/wav`.
+
+Below is a code example, illustrating the response of user text along with an audio byte array, using the `gpt-4o-audio-preview` model:
+
+[source,java]
+----
+var userMessage = new UserMessage("Tell me joke about Spring Framework");
+
+ChatResponse response = chatModel.call(new Prompt(List.of(userMessage),
+        OpenAiChatOptions.builder()
+            .withModel(OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW)
+            .withOutputModalities(List.of("text", "audio"))
+            .withOutputAudio(new AudioParameters(Voice.ALLOY, AudioResponseFormat.WAV))
+            .build()));
+
+String text = response.getResult().getOutput().getContent(); // audio transcript
+
+byte[] waveAudio = response.getResult().getOutput().getMedia().get(0).getDataAsByteArray(); // audio data
+----
+
+You have to specify an `audio` modality in the `OpenAiChatOptions` to generate audio output. 
+The `AudioParameters` class provides the voice and audio format for the audio output.
+
 == Structured Outputs
 
 OpenAI provides custom https://platform.openai.com/docs/guides/structured-outputs[Structured Outputs] APIs that ensure your model generates responses conforming strictly to your provided `JSON Schema`. 

spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/additional-spring-configuration-metadata.json

@@ -10,6 +10,11 @@
 	  "name": "spring.ai.mistralai.chat.options.tool-choice",
 	  "type": "org.springframework.ai.mistralai.api.MistralAiApi$ChatCompletionRequest$ToolChoice",
 	  "sourceType": "org.springframework.ai.mistralai.MistralAiChatOptions"
+	},
+	{
+	  "name": "spring.ai.openai.chat.output-audio",
+	  "type": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters",
+	  "sourceType": "org.springframework.ai.openai.OpenAiChatOptions"
 	}
   ],
   "properties": [
@@ -22,6 +27,16 @@
 	  "name": "spring.ai.azure.openai.chat.options.enhancements.ocr",
 	  "type": "com.azure.ai.openai.models.AzureChatOCREnhancementConfiguration",
 	  "sourceType": "com.azure.ai.openai.models.AzureChatEnhancementConfiguration"
+	},
+	{
+	  "name": "spring.ai.openai.chat.output-audio.voice",
+	  "type": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters$Voice",
+	  "sourceType": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters"
+	},
+	{
+	  "name": "spring.ai.openai.chat.output-audio.format",
+	  "type": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters$AudioResponseFormat",
+	  "sourceType": "org.springframework.ai.openai.api.OpenAiApi$ChatCompletionRequest$AudioParameters"
 	}
   ],
   "hints": []

spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java

@@ -38,6 +38,7 @@ import org.springframework.ai.openai.OpenAiAudioTranscriptionModel;
 import org.springframework.ai.openai.OpenAiChatModel;
 import org.springframework.ai.openai.OpenAiEmbeddingModel;
 import org.springframework.ai.openai.OpenAiImageModel;
+import org.springframework.ai.openai.api.OpenAiApi;
 import org.springframework.boot.autoconfigure.AutoConfigurations;
 import org.springframework.boot.test.context.runner.ApplicationContextRunner;
 import org.springframework.core.io.ClassPathResource;
@@ -55,7 +56,7 @@ public class OpenAiAutoConfigurationIT {
 		.withConfiguration(AutoConfigurations.of(OpenAiAutoConfiguration.class));
 
 	@Test
-	void generate() {
+	void chatCall() {
 		this.contextRunner.run(context -> {
 			OpenAiChatModel chatModel = context.getBean(OpenAiChatModel.class);
 			String response = chatModel.call("Hello");
@@ -64,6 +65,25 @@ public class OpenAiAutoConfigurationIT {
 		});
 	}
 
+	@Test
+	void chatCallAudioResponse() {
+		this.contextRunner
+			.withPropertyValues(
+					"spring.ai.openai.chat.options.model=" + OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue(),
+					"spring.ai.openai.chat.options.output-modalities=text,audio",
+					"spring.ai.openai.chat.options.output-audio.voice=ALLOY",
+					"spring.ai.openai.chat.options.output-audio.format=WAV")
+			.run(context -> {
+				OpenAiChatModel chatModel = context.getBean(OpenAiChatModel.class);
+
+				ChatResponse response = chatModel
+					.call(new Prompt(new UserMessage("Tell me joke about Spring Framework")));
+				assertThat(response).isNotNull();
+				logger.info("Response: " + response);
+				// AudioPlayer.play(response.getResult().getOutput().getMedia().get(0).getDataAsByteArray());
+			});
+	}
+
 	@Test
 	void transcribe() {
 		this.contextRunner.run(context -> {

spring-ai-test/src/main/java/org/springframework/ai/utils/AudioPlayer.java

@@ -0,0 +1,65 @@
+/*
+* Copyright 2024 - 2024 the original author or authors.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* https://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.springframework.ai.utils;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.FileInputStream;
+import java.io.InputStream;
+
+import javax.sound.sampled.AudioInputStream;
+import javax.sound.sampled.AudioSystem;
+import javax.sound.sampled.Clip;
+
+/**
+ * @author Christian Tzolov
+ * @since 1.0.0
+ */
+public class AudioPlayer {
+
+	public static void main(String[] args) throws Exception {
+
+		play(new BufferedInputStream(new FileInputStream(
+				"/Users/christiantzolov/Dev/projects/spring-ai/models/spring-ai-openai/output.wav")));
+	}
+
+	public static void play(byte[] data) {
+		play(new BufferedInputStream(new ByteArrayInputStream(data)));
+	}
+
+	public static void play(InputStream data) {
+
+		try {
+			try (AudioInputStream audio = AudioSystem.getAudioInputStream(data); Clip clip = AudioSystem.getClip()) {
+				clip.open(audio);
+				clip.start();
+				// wait to start
+				while (!clip.isRunning()) {
+					Thread.sleep(100);
+				}
+				// wait to finish
+				while (clip.isRunning()) {
+					Thread.sleep(3000);
+				}
+			}
+		}
+		catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+
+	}
+
+}