diff --git a/models/spring-ai-openai/README.md b/models/spring-ai-openai/README.md index fc6a2361e..a45db418a 100644 --- a/models/spring-ai-openai/README.md +++ b/models/spring-ai-openai/README.md @@ -2,3 +2,6 @@ [OpenAI Embedding Documentation](https://docs.spring.io/spring-ai/reference/api/embeddings/openai-embeddings.html) +[OpenAI Image Generation](https://docs.spring.io/spring-ai/reference/api/clients/image/openai-image.html) + +[OpenAI Transcription Generation](TODO) \ No newline at end of file diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java index 6d986f379..a81dacd4f 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java @@ -30,9 +30,11 @@ import org.springframework.ai.model.ModelOptionsUtils; import org.springframework.ai.openai.api.common.ApiUtils; import org.springframework.boot.context.properties.bind.ConstructorBinding; import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; +import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestClient; import org.springframework.web.reactive.function.client.WebClient; @@ -42,6 +44,7 @@ import org.springframework.web.reactive.function.client.WebClient; * OpenAI Embedding API: https://platform.openai.com/docs/api-reference/embeddings. * * @author Christian Tzolov + * @author Michael Lavelle */ public class OpenAiApi { @@ -50,6 +53,9 @@ public class OpenAiApi { private static final Predicate SSE_DONE_PREDICATE = "[DONE]"::equals; private final RestClient restClient; + + private final RestClient multipartRestClient; + private final WebClient webClient; /** @@ -86,6 +92,15 @@ public class OpenAiApi { .defaultStatusHandler(ApiUtils.DEFAULT_RESPONSE_ERROR_HANDLER) .build(); + this.multipartRestClient = restClientBuilder + .baseUrl(baseUrl) + .defaultHeaders(multipartFormDataHeaders -> { + multipartFormDataHeaders.setBearerAuth(openAiToken); + multipartFormDataHeaders.setContentType(MediaType.MULTIPART_FORM_DATA); + }) + .defaultStatusHandler(ApiUtils.DEFAULT_RESPONSE_ERROR_HANDLER) + .build(); + this.webClient = WebClient.builder() .baseUrl(baseUrl) .defaultHeaders(ApiUtils.getJsonContentHeaders(openAiToken)) @@ -97,7 +112,7 @@ public class OpenAiApi { * GPT-4 and GPT-4 Turbo and * GPT-3.5 Turbo. */ - enum ChatModel { + public enum ChatModel { /** * (New) GPT-4 Turbo - latest GPT-4 model intended to reduce cases * of “laziness” where the model doesn’t complete a task. @@ -169,42 +184,6 @@ public class OpenAiApi { } } - /** - * OpenAI Embeddings Models: - * Embeddings. - */ - enum EmbeddingModel { - - /** - * Most capable embedding model for both english and non-english tasks. - * DIMENSION: 3072 - */ - TEXT_EMBEDDING_3_LARGE("text-embedding-3-large"), - - /** - * Increased performance over 2nd generation ada embedding model. - * DIMENSION: 1536 - */ - TEXT_EMBEDDING_3_SMALL("text-embedding-3-small"), - - /** - * Most capable 2nd generation embedding model, replacing 16 first - * generation models. - * DIMENSION: 1536 - */ - TEXT_EMBEDDING_ADA_002("text-embedding-ada-002"); - - public final String value; - - EmbeddingModel(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - } - /** * Represents a tool the model may call. Currently, only functions are supported as a tool. * @@ -708,6 +687,44 @@ public class OpenAiApi { .map(content -> ModelOptionsUtils.jsonToObject(content, ChatCompletionChunk.class)); } + // Embeddings API + + /** + * OpenAI Embeddings Models: + * Embeddings. + */ + public enum EmbeddingModel { + + /** + * Most capable embedding model for both english and non-english tasks. + * DIMENSION: 3072 + */ + TEXT_EMBEDDING_3_LARGE("text-embedding-3-large"), + + /** + * Increased performance over 2nd generation ada embedding model. + * DIMENSION: 1536 + */ + TEXT_EMBEDDING_3_SMALL("text-embedding-3-small"), + + /** + * Most capable 2nd generation embedding model, replacing 16 first + * generation models. + * DIMENSION: 1536 + */ + TEXT_EMBEDDING_ADA_002("text-embedding-ada-002"); + + public final String value; + + EmbeddingModel(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + } + /** * Represents an embedding vector returned by embedding endpoint. * @@ -824,5 +841,87 @@ public class OpenAiApi { .toEntity(new ParameterizedTypeReference<>() { }); } + + // Transcription API + + // @JsonInclude(Include.NON_NULL) + // public record Transcription( + // @JsonProperty("text") String text) { + // } + + // /** + // * + // * @param model ID of the model to use. + // * @param language The language of the input audio. Supplying the input language in ISO-639-1 format will improve accuracy and latency. + // * @param prompt An optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language. + // * @param responseFormat An object specifying the format that the model must output. + // * @param temperature What sampling temperature to use, between 0 and 1. Higher values like 0.8 will make the output + // * more random, while lower values like 0.2 will make it more focused and deterministic. */ + // @JsonInclude(Include.NON_NULL) + // public record TranscriptionRequest ( + // @JsonProperty("model") String model, + // @JsonProperty("language") String language, + // @JsonProperty("prompt") String prompt, + // @JsonProperty("response_format") ResponseFormat responseFormat, + // @JsonProperty("temperature") Float temperature) { + + // /** + // * Shortcut constructor for a transcription request with the given model and temperature + // * + // * @param model ID of the model to use. + // * @param temperature What sampling temperature to use, between 0 and 1. + // */ + // public TranscriptionRequest(String model, Float temperature) { + // this(model, null, null, null, temperature); + // } + + // public TranscriptionRequest() { + // this(null, null, null, null, null); + // } + + // /** + // * An object specifying the format that the model must output. + // * @param type Must be one of 'text' or 'json_object'. + // */ + // @JsonInclude(Include.NON_NULL) + // public record ResponseFormat( + // @JsonProperty("type") String type) { + // } + // } + + // /** + // * Creates a model response for the given transcription. + // * + // * @param transcriptionRequest The transcription request. + // * @return Entity response with {@link Transcription} as a body and HTTP status code and headers. + // */ + // public ResponseEntity transcriptionEntityJson(MultiValueMap transcriptionRequest) { + + // Assert.notNull(transcriptionRequest, "The request body can not be null."); + + // return this.multipartRestClient.post() + // .uri("/v1/audio/transcriptions") + // .body(transcriptionRequest) + // .retrieve() + // .toEntity(Transcription.class); + // } + + // /** + // * Creates a model response for the given transcription. + // * + // * @param transcriptionRequest The transcription request. + // * @return Entity response with {@link String} as a body and HTTP status code and headers. + // */ + // public ResponseEntity transcriptionEntityText(MultiValueMap transcriptionRequest) { + + // Assert.notNull(transcriptionRequest, "The request body can not be null."); + + // return this.multipartRestClient.post() + // .uri("/v1/audio/transcriptions") + // .body(transcriptionRequest) + // .accept(MediaType.TEXT_PLAIN) + // .retrieve() + // .toEntity(String.class); + // } } // @formatter:on diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiAudioApi.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiAudioApi.java index 52c38f13b..e4fbc9dc3 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiAudioApi.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiAudioApi.java @@ -280,7 +280,7 @@ public class OpenAiAudioApi { @JsonProperty("model") String model, @JsonProperty("language") String language, @JsonProperty("prompt") String prompt, - @JsonProperty("response_format") TextualResponseFormat responseFormat, + @JsonProperty("response_format") TranscriptResponseFormat responseFormat, @JsonProperty("temperature") Float temperature, @JsonProperty("timestamp_granularities") GranularityType granularityType) { // @formatter:on @@ -318,7 +318,7 @@ public class OpenAiAudioApi { private String prompt; - private TextualResponseFormat responseFormat = TextualResponseFormat.JSON; + private TranscriptResponseFormat responseFormat = TranscriptResponseFormat.JSON; private Float temperature; @@ -344,7 +344,7 @@ public class OpenAiAudioApi { return this; } - public Builder withResponseFormat(TextualResponseFormat response_format) { + public Builder withResponseFormat(TranscriptResponseFormat response_format) { this.responseFormat = response_format; return this; } @@ -375,7 +375,7 @@ public class OpenAiAudioApi { * The format of the transcript and translation outputs, in one of these options: * json, text, srt, verbose_json, or vtt. Defaults to json. */ - public enum TextualResponseFormat { + public enum TranscriptResponseFormat { // @formatter:off @JsonProperty("json") JSON("json", StructuredResponse.class), @@ -393,7 +393,7 @@ public class OpenAiAudioApi { return this == JSON || this == VERBOSE_JSON; } - TextualResponseFormat(String value, Class responseType) { + TranscriptResponseFormat(String value, Class responseType) { this.value = value; this.responseType = responseType; } @@ -429,7 +429,7 @@ public class OpenAiAudioApi { @JsonProperty("file") byte[] file, @JsonProperty("model") String model, @JsonProperty("prompt") String prompt, - @JsonProperty("response_format") TextualResponseFormat responseFormat, + @JsonProperty("response_format") TranscriptResponseFormat responseFormat, @JsonProperty("temperature") Float temperature) { // @formatter:on @@ -445,7 +445,7 @@ public class OpenAiAudioApi { private String prompt; - private TextualResponseFormat responseFormat = TextualResponseFormat.JSON; + private TranscriptResponseFormat responseFormat = TranscriptResponseFormat.JSON; private Float temperature; @@ -464,7 +464,7 @@ public class OpenAiAudioApi { return this; } - public Builder withResponseFormat(TextualResponseFormat responseFormat) { + public Builder withResponseFormat(TranscriptResponseFormat responseFormat) { this.responseFormat = responseFormat; return this; } @@ -601,7 +601,7 @@ public class OpenAiAudioApi { multipartBody.add("response_format", requestBody.responseFormat().getValue()); multipartBody.add("temperature", requestBody.temperature()); if (requestBody.granularityType() != null) { - Assert.isTrue(requestBody.responseFormat() == TextualResponseFormat.VERBOSE_JSON, + Assert.isTrue(requestBody.responseFormat() == TranscriptResponseFormat.VERBOSE_JSON, "response_format must be set to verbose_json to use timestamp granularities."); multipartBody.add("timestamp_granularities[]", requestBody.granularityType().getValue()); } diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscription.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscription.java new file mode 100644 index 000000000..b0f313c35 --- /dev/null +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscription.java @@ -0,0 +1,76 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai.audio.transcription; + +import org.springframework.ai.model.ModelResult; +import org.springframework.ai.openai.metadata.audio.OpenAiAudioTranscriptionMetadata; +import org.springframework.lang.Nullable; + +import java.util.Objects; + +/** + * Represents a response returned by the AI. + * + * @author Michael Lavelle + * @since 0.8.1 + */ +public class AudioTranscription implements ModelResult { + + private String text; + + private OpenAiAudioTranscriptionMetadata transcriptionMetadata; + + public AudioTranscription(String text) { + this.text = text; + } + + @Override + public String getOutput() { + return this.text; + } + + @Override + public OpenAiAudioTranscriptionMetadata getMetadata() { + return transcriptionMetadata != null ? transcriptionMetadata : OpenAiAudioTranscriptionMetadata.NULL; + } + + public AudioTranscription withTranscriptionMetadata( + @Nullable OpenAiAudioTranscriptionMetadata transcriptionMetadata) { + this.transcriptionMetadata = transcriptionMetadata; + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof AudioTranscription that)) + return false; + return Objects.equals(text, that.text) && Objects.equals(transcriptionMetadata, that.transcriptionMetadata); + } + + @Override + public int hashCode() { + return Objects.hash(text, transcriptionMetadata); + } + + @Override + public String toString() { + return "Transcript{" + "text=" + text + ", transcriptionMetadata=" + transcriptionMetadata + '}'; + } + +} diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscriptionRequest.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscriptionRequest.java new file mode 100644 index 000000000..a1c549f85 --- /dev/null +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscriptionRequest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai.audio.transcription; + +import org.springframework.ai.model.ModelOptions; +import org.springframework.ai.model.ModelRequest; +import org.springframework.core.io.Resource; + +/** + * @author Michael Lavelle + * @since 0.8.1 + */ +public class AudioTranscriptionRequest implements ModelRequest { + + private Resource audioResource; + + private ModelOptions modelOptions; + + public AudioTranscriptionRequest(Resource audioResource) { + this.audioResource = audioResource; + } + + public AudioTranscriptionRequest(Resource audioResource, ModelOptions modelOptions) { + this.audioResource = audioResource; + this.modelOptions = modelOptions; + } + + @Override + public Resource getInstructions() { + return audioResource; + } + + @Override + public ModelOptions getOptions() { + return modelOptions; + } + +} diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscriptionResponse.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscriptionResponse.java new file mode 100644 index 000000000..94da2cbe5 --- /dev/null +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/AudioTranscriptionResponse.java @@ -0,0 +1,60 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai.audio.transcription; + +import org.springframework.ai.model.ModelResponse; +import org.springframework.ai.openai.metadata.audio.OpenAiAudioTranscriptionResponseMetadata; + +import java.util.Arrays; +import java.util.List; + +/** + * @author Michael Lavelle + * @since 0.8.1 + */ +public class AudioTranscriptionResponse implements ModelResponse { + + private AudioTranscription transcript; + + private OpenAiAudioTranscriptionResponseMetadata transcriptionResponseMetadata; + + public AudioTranscriptionResponse(AudioTranscription transcript) { + this(transcript, OpenAiAudioTranscriptionResponseMetadata.NULL); + } + + public AudioTranscriptionResponse(AudioTranscription transcript, + OpenAiAudioTranscriptionResponseMetadata transcriptionResponseMetadata) { + this.transcript = transcript; + this.transcriptionResponseMetadata = transcriptionResponseMetadata; + } + + @Override + public AudioTranscription getResult() { + return transcript; + } + + @Override + public List getResults() { + return Arrays.asList(transcript); + } + + @Override + public OpenAiAudioTranscriptionResponseMetadata getMetadata() { + return transcriptionResponseMetadata; + } + +} diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/OpenAiAudioTranscriptionClient.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/OpenAiAudioTranscriptionClient.java new file mode 100644 index 000000000..9c51e4f35 --- /dev/null +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/OpenAiAudioTranscriptionClient.java @@ -0,0 +1,185 @@ +/* +* Copyright 2024-2024 the original author or authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.springframework.ai.openai.audio.transcription; + +import java.time.Duration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.ai.chat.metadata.RateLimit; +import org.springframework.ai.model.ModelClient; +import org.springframework.ai.openai.api.OpenAiAudioApi; +import org.springframework.ai.openai.api.OpenAiAudioApi.StructuredResponse; +import org.springframework.ai.openai.api.common.OpenAiApiException; +import org.springframework.ai.openai.metadata.audio.OpenAiAudioTranscriptionResponseMetadata; +import org.springframework.ai.openai.metadata.support.OpenAiResponseHeaderExtractor; +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.Assert; + +/** + * OpenAI audio transcription client implementation for backed by {@link OpenAiAudioApi}. + * + * @author Michael Lavelle + * @author Christian Tzolov + * @see OpenAiAudioApi + * @since 0.8.1 + */ +public class OpenAiAudioTranscriptionClient + implements ModelClient { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private final OpenAiAudioTranscriptionOptions defaultOptions; + + public final RetryTemplate retryTemplate = RetryTemplate.builder() + .maxAttempts(10) + .retryOn(OpenAiApiException.class) + .exponentialBackoff(Duration.ofMillis(2000), 5, Duration.ofMillis(3 * 60000)) + .build(); + + private final OpenAiAudioApi audioApi; + + public OpenAiAudioTranscriptionClient(OpenAiAudioApi audioApi) { + this(audioApi, + OpenAiAudioTranscriptionOptions.builder() + .withModel(OpenAiAudioApi.WhisperModel.WHISPER_1.getValue()) + .withResponseFormat(OpenAiAudioApi.TranscriptResponseFormat.JSON) + .withTemperature(0.7f) + .build()); + } + + public OpenAiAudioTranscriptionClient(OpenAiAudioApi audioApi, OpenAiAudioTranscriptionOptions options) { + Assert.notNull(audioApi, "OpenAiAudioApi must not be null"); + Assert.notNull(options, "OpenAiTranscriptionOptions must not be null"); + this.audioApi = audioApi; + this.defaultOptions = options; + } + + public String call(Resource audioResource) { + AudioTranscriptionRequest transcriptionRequest = new AudioTranscriptionRequest(audioResource); + return call(transcriptionRequest).getResult().getOutput(); + } + + @Override + public AudioTranscriptionResponse call(AudioTranscriptionRequest request) { + + return this.retryTemplate.execute(ctx -> { + + Resource audioResource = request.getInstructions(); + + OpenAiAudioApi.TranscriptionRequest requestBody = createRequestBody(request); + + if (requestBody.responseFormat().isJsonType()) { + + ResponseEntity transcriptionEntity = this.audioApi.createTranscription(requestBody, + StructuredResponse.class); + + var transcription = transcriptionEntity.getBody(); + + if (transcription == null) { + logger.warn("No transcription returned for request: {}", audioResource); + return new AudioTranscriptionResponse(null); + } + + AudioTranscription transcript = new AudioTranscription(transcription.text()); + + RateLimit rateLimits = OpenAiResponseHeaderExtractor.extractAiResponseHeaders(transcriptionEntity); + + return new AudioTranscriptionResponse(transcript, + OpenAiAudioTranscriptionResponseMetadata.from(transcriptionEntity.getBody()) + .withRateLimit(rateLimits)); + + } + else { + + ResponseEntity transcriptionEntity = this.audioApi.createTranscription(requestBody, + String.class); + + var transcription = transcriptionEntity.getBody(); + + if (transcription == null) { + logger.warn("No transcription returned for request: {}", audioResource); + return new AudioTranscriptionResponse(null); + } + + AudioTranscription transcript = new AudioTranscription(transcription); + + RateLimit rateLimits = OpenAiResponseHeaderExtractor.extractAiResponseHeaders(transcriptionEntity); + + return new AudioTranscriptionResponse(transcript, + OpenAiAudioTranscriptionResponseMetadata.from(transcriptionEntity.getBody()) + .withRateLimit(rateLimits)); + } + }); + } + + OpenAiAudioApi.TranscriptionRequest createRequestBody(AudioTranscriptionRequest request) { + + OpenAiAudioTranscriptionOptions options = this.defaultOptions; + + if (request.getOptions() != null) { + if (request.getOptions() instanceof OpenAiAudioTranscriptionOptions runtimeOptions) { + options = this.merge(options, runtimeOptions); + } + else { + throw new IllegalArgumentException("Prompt options are not of type TranscriptionOptions: " + + request.getOptions().getClass().getSimpleName()); + } + } + + OpenAiAudioApi.TranscriptionRequest audioTranscriptionRequest = OpenAiAudioApi.TranscriptionRequest.builder() + .withFile(toBytes(request.getInstructions())) + .withResponseFormat(options.getResponseFormat()) + .withPrompt(options.getPrompt()) + .withTemperature(options.getTemperature()) + .withLanguage(options.getLanguage()) + .withModel(options.getModel()) + .build(); + + return audioTranscriptionRequest; + } + + private byte[] toBytes(Resource resource) { + try { + return resource.getInputStream().readAllBytes(); + } + catch (Exception e) { + throw new IllegalArgumentException("Failed to read resource: " + resource, e); + } + } + + private OpenAiAudioTranscriptionOptions merge(OpenAiAudioTranscriptionOptions source, + OpenAiAudioTranscriptionOptions target) { + + if (source == null) { + source = new OpenAiAudioTranscriptionOptions(); + } + + OpenAiAudioTranscriptionOptions merged = new OpenAiAudioTranscriptionOptions(); + merged.setLanguage(source.getLanguage() != null ? source.getLanguage() : target.getLanguage()); + merged.setModel(source.getModel() != null ? source.getModel() : target.getModel()); + merged.setPrompt(source.getPrompt() != null ? source.getPrompt() : target.getPrompt()); + merged.setResponseFormat( + source.getResponseFormat() != null ? source.getResponseFormat() : target.getResponseFormat()); + merged.setTemperature(source.getTemperature() != null ? source.getTemperature() : target.getTemperature()); + return merged; + } + +} diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/OpenAiAudioTranscriptionOptions.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/OpenAiAudioTranscriptionOptions.java new file mode 100644 index 000000000..9e66dc5f0 --- /dev/null +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/audio/transcription/OpenAiAudioTranscriptionOptions.java @@ -0,0 +1,206 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai.audio.transcription; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + +import org.springframework.ai.model.ModelOptions; +import org.springframework.ai.openai.api.OpenAiAudioApi.TranscriptResponseFormat; +import org.springframework.ai.openai.api.OpenAiAudioApi.TranscriptionRequest.GranularityType; + +/** + * @author Michael Lavelle + * @author Christian Tzolov + * @since 0.8.1 + */ +@JsonInclude(Include.NON_NULL) +public class OpenAiAudioTranscriptionOptions implements ModelOptions { + + // @formatter:off + /** + * ID of the model to use. + */ + private @JsonProperty("model") String model; + + /** + * An object specifying the format that the model must output. Setting to { "type": + * "json_object" } enables JSON mode, which guarantees the message the model generates is valid JSON. + */ + private @JsonProperty("response_format") TranscriptResponseFormat responseFormat; + + private @JsonProperty("prompt") String prompt; + + private @JsonProperty("language") String language; + + /** + * What sampling temperature to use, between 0 and 1. Higher values like 0.8 will make the output + * more random, while lower values like 0.2 will make it more focused and deterministic. + */ + private @JsonProperty("temperature") Float temperature; + + private @JsonProperty("timestamp_granularities") GranularityType granularityType; + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + protected OpenAiAudioTranscriptionOptions options; + + public Builder() { + this.options = new OpenAiAudioTranscriptionOptions(); + } + + public Builder(OpenAiAudioTranscriptionOptions options) { + this.options = options; + } + + public Builder withModel(String model) { + this.options.model = model; + return this; + } + + public Builder withLanguage(String language) { + this.options.language = language; + return this; + } + + public Builder withPrompt(String prompt) { + this.options.prompt = prompt; + return this; + } + + public Builder withResponseFormat(TranscriptResponseFormat responseFormat) { + this.options.responseFormat = responseFormat; + return this; + } + + public Builder withTemperature(Float temperature) { + this.options.temperature = temperature; + return this; + } + + public Builder withGranularityType(GranularityType granularityType) { + this.options.granularityType = granularityType; + return this; + } + + public OpenAiAudioTranscriptionOptions build() { + return this.options; + } + + } + + public String getModel() { + return this.model; + } + + public void setModel(String model) { + this.model = model; + } + + public String getLanguage() { + return this.language; + } + + public void setLanguage(String language) { + this.language = language; + } + + public String getPrompt() { + return this.prompt; + } + + public void setPrompt(String prompt) { + this.prompt = prompt; + } + + public Float getTemperature() { + return this.temperature; + } + + public void setTemperature(Float temperature) { + this.temperature = temperature; + } + + + public TranscriptResponseFormat getResponseFormat() { + return this.responseFormat; + } + + public void setResponseFormat(TranscriptResponseFormat responseFormat) { + this.responseFormat = responseFormat; + } + + public GranularityType getGranularityType() { + return this.granularityType; + } + + public void setGranularityType(GranularityType granularityType) { + this.granularityType = granularityType; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((model == null) ? 0 : model.hashCode()); + result = prime * result + ((prompt == null) ? 0 : prompt.hashCode()); + result = prime * result + ((language == null) ? 0 : language.hashCode()); + result = prime * result + ((responseFormat == null) ? 0 : responseFormat.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + OpenAiAudioTranscriptionOptions other = (OpenAiAudioTranscriptionOptions) obj; + if (this.model == null) { + if (other.model != null) + return false; + } + else if (!model.equals(other.model)) + return false; + if (this.prompt == null) { + if (other.prompt != null) + return false; + } + else if (!this.prompt.equals(other.prompt)) + return false; + if (this.language == null) { + if (other.language != null) + return false; + } + else if (!this.language.equals(other.language)) + return false; + if (this.responseFormat == null) { + if (other.responseFormat != null) + return false; + } + else if (!this.responseFormat.equals(other.responseFormat)) + return false; + return true; + } +} diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/audio/OpenAiAudioTranscriptionMetadata.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/audio/OpenAiAudioTranscriptionMetadata.java new file mode 100644 index 000000000..306e6f31d --- /dev/null +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/audio/OpenAiAudioTranscriptionMetadata.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +* @author Michael Lavelle +* @since 0.8.1 +*/ +package org.springframework.ai.openai.metadata.audio; + +import org.springframework.ai.model.ResultMetadata; + +public interface OpenAiAudioTranscriptionMetadata extends ResultMetadata { + + OpenAiAudioTranscriptionMetadata NULL = OpenAiAudioTranscriptionMetadata.create(); + + /** + * Factory method used to construct a new {@link OpenAiAudioTranscriptionMetadata} + * @return a new {@link OpenAiAudioTranscriptionMetadata} + */ + static OpenAiAudioTranscriptionMetadata create() { + return new OpenAiAudioTranscriptionMetadata() { + }; + } + +} diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/audio/OpenAiAudioTranscriptionResponseMetadata.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/audio/OpenAiAudioTranscriptionResponseMetadata.java new file mode 100644 index 000000000..94ee89ef6 --- /dev/null +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/audio/OpenAiAudioTranscriptionResponseMetadata.java @@ -0,0 +1,80 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai.metadata.audio; + +import org.springframework.ai.chat.metadata.EmptyRateLimit; +import org.springframework.ai.chat.metadata.RateLimit; +import org.springframework.ai.model.ResponseMetadata; +import org.springframework.ai.openai.api.OpenAiAudioApi; +import org.springframework.ai.openai.metadata.OpenAiRateLimit; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Audio transcription metadata implementation for {@literal OpenAI}. + * + * @author MichaelLavelle + * @since 0.8.1 + * @see RateLimit + */ +public class OpenAiAudioTranscriptionResponseMetadata implements ResponseMetadata { + + protected static final String AI_METADATA_STRING = "{ @type: %1$s, rateLimit: %4$s }"; + + public static final OpenAiAudioTranscriptionResponseMetadata NULL = new OpenAiAudioTranscriptionResponseMetadata() { + }; + + public static OpenAiAudioTranscriptionResponseMetadata from(OpenAiAudioApi.StructuredResponse result) { + Assert.notNull(result, "OpenAI Transcription must not be null"); + OpenAiAudioTranscriptionResponseMetadata transcriptionResponseMetadata = new OpenAiAudioTranscriptionResponseMetadata(); + return transcriptionResponseMetadata; + } + + public static OpenAiAudioTranscriptionResponseMetadata from(String result) { + Assert.notNull(result, "OpenAI Transcription must not be null"); + OpenAiAudioTranscriptionResponseMetadata transcriptionResponseMetadata = new OpenAiAudioTranscriptionResponseMetadata(); + return transcriptionResponseMetadata; + } + + @Nullable + private RateLimit rateLimit; + + protected OpenAiAudioTranscriptionResponseMetadata() { + this(null); + } + + protected OpenAiAudioTranscriptionResponseMetadata(@Nullable OpenAiRateLimit rateLimit) { + this.rateLimit = rateLimit; + } + + @Nullable + public RateLimit getRateLimit() { + RateLimit rateLimit = this.rateLimit; + return rateLimit != null ? rateLimit : new EmptyRateLimit(); + } + + public OpenAiAudioTranscriptionResponseMetadata withRateLimit(RateLimit rateLimit) { + this.rateLimit = rateLimit; + return this; + } + + @Override + public String toString() { + return AI_METADATA_STRING.formatted(getClass().getName(), getRateLimit()); + } + +} diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/support/OpenAiResponseHeaderExtractor.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/support/OpenAiResponseHeaderExtractor.java index 21c286522..ebcec840a 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/support/OpenAiResponseHeaderExtractor.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/support/OpenAiResponseHeaderExtractor.java @@ -52,7 +52,7 @@ public class OpenAiResponseHeaderExtractor { private static final Logger logger = LoggerFactory.getLogger(OpenAiResponseHeaderExtractor.class); - public static RateLimit extractAiResponseHeaders(ResponseEntity response) { + public static RateLimit extractAiResponseHeaders(ResponseEntity response) { Long requestsLimit = getHeaderAsLong(response, REQUESTS_LIMIT_HEADER.getName()); Long requestsRemaining = getHeaderAsLong(response, REQUESTS_REMAINING_HEADER.getName()); @@ -66,7 +66,7 @@ public class OpenAiResponseHeaderExtractor { tokensReset); } - private static Duration getHeaderAsDuration(ResponseEntity response, String headerName) { + private static Duration getHeaderAsDuration(ResponseEntity response, String headerName) { var headers = response.getHeaders(); if (headers.containsKey(headerName)) { var values = headers.get(headerName); @@ -77,7 +77,7 @@ public class OpenAiResponseHeaderExtractor { return null; } - private static Long getHeaderAsLong(ResponseEntity response, String headerName) { + private static Long getHeaderAsLong(ResponseEntity response, String headerName) { var headers = response.getHeaders(); if (headers.containsKey(headerName)) { var values = headers.get(headerName); diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java index d43a25dde..3d61368f3 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java @@ -2,7 +2,9 @@ package org.springframework.ai.openai; import org.springframework.ai.embedding.EmbeddingClient; import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.ai.openai.api.OpenAiAudioApi; import org.springframework.ai.openai.api.OpenAiImageApi; +import org.springframework.ai.openai.audio.transcription.OpenAiAudioTranscriptionClient; import org.springframework.boot.SpringBootConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.util.StringUtils; @@ -20,6 +22,11 @@ public class OpenAiTestConfiguration { return new OpenAiImageApi(getApiKey()); } + @Bean + public OpenAiAudioApi openAiAudioApi() { + return new OpenAiAudioApi(getApiKey()); + } + private String getApiKey() { String apiKey = System.getenv("OPENAI_API_KEY"); if (!StringUtils.hasText(apiKey)) { @@ -35,6 +42,12 @@ public class OpenAiTestConfiguration { return openAiChatClient; } + @Bean + public OpenAiAudioTranscriptionClient openAiTranscriptionClient(OpenAiAudioApi api) { + OpenAiAudioTranscriptionClient openAiTranscriptionClient = new OpenAiAudioTranscriptionClient(api); + return openAiTranscriptionClient; + } + @Bean public OpenAiImageClient openAiImageClient(OpenAiImageApi imageApi) { OpenAiImageClient openAiImageClient = new OpenAiImageClient(imageApi); diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/OpenAiTranscriptionClientIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/OpenAiTranscriptionClientIT.java new file mode 100644 index 000000000..98e2608d6 --- /dev/null +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/OpenAiTranscriptionClientIT.java @@ -0,0 +1,53 @@ +package org.springframework.ai.openai.audio.transcription; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +import org.springframework.ai.openai.OpenAiTestConfiguration; +import org.springframework.ai.openai.api.OpenAiAudioApi; +import org.springframework.ai.openai.api.OpenAiAudioApi.TranscriptResponseFormat; +import org.springframework.ai.openai.audio.transcription.AudioTranscriptionRequest; +import org.springframework.ai.openai.audio.transcription.AudioTranscriptionResponse; +import org.springframework.ai.openai.testutils.AbstractIT; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.io.Resource; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(classes = OpenAiTestConfiguration.class) +@EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+") +class OpenAiTranscriptionClientIT extends AbstractIT { + + @Value("classpath:/speech/jfk.flac") + private Resource audioFile; + + @Test + void transcriptionTest() { + OpenAiAudioTranscriptionOptions transcriptionOptions = OpenAiAudioTranscriptionOptions.builder() + .withResponseFormat(TranscriptResponseFormat.TEXT) + .withTemperature(0f) + .build(); + AudioTranscriptionRequest transcriptionRequest = new AudioTranscriptionRequest(audioFile, transcriptionOptions); + AudioTranscriptionResponse response = openAiTranscriptionClient.call(transcriptionRequest); + assertThat(response.getResults()).hasSize(1); + assertThat(response.getResults().get(0).getOutput().toLowerCase().contains("fellow")).isTrue(); + } + + @Test + void transcriptionTestWithOptions() { + OpenAiAudioApi.TranscriptResponseFormat responseFormat = OpenAiAudioApi.TranscriptResponseFormat.VTT; + + OpenAiAudioTranscriptionOptions transcriptionOptions = OpenAiAudioTranscriptionOptions.builder() + .withLanguage("en") + .withPrompt("Ask not this, but ask that") + .withTemperature(0f) + .withResponseFormat(responseFormat) + .build(); + AudioTranscriptionRequest transcriptionRequest = new AudioTranscriptionRequest(audioFile, transcriptionOptions); + AudioTranscriptionResponse response = openAiTranscriptionClient.call(transcriptionRequest); + assertThat(response.getResults()).hasSize(1); + assertThat(response.getResults().get(0).getOutput().toLowerCase().contains("fellow")).isTrue(); + } + +} diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/OpenAiTranscriptionClientWithTranscriptionResponseMetadataTests.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/OpenAiTranscriptionClientWithTranscriptionResponseMetadataTests.java new file mode 100644 index 000000000..2d7e567d4 --- /dev/null +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/OpenAiTranscriptionClientWithTranscriptionResponseMetadataTests.java @@ -0,0 +1,166 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai.audio.transcription; + +import java.time.Duration; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.chat.metadata.RateLimit; +import org.springframework.ai.openai.api.OpenAiAudioApi; +import org.springframework.ai.openai.audio.transcription.AudioTranscriptionRequest; +import org.springframework.ai.openai.audio.transcription.AudioTranscriptionResponse; +import org.springframework.ai.openai.metadata.audio.OpenAiAudioTranscriptionMetadata; +import org.springframework.ai.openai.metadata.audio.OpenAiAudioTranscriptionResponseMetadata; +import org.springframework.ai.openai.metadata.support.OpenAiApiResponseHeaders; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.autoconfigure.web.client.RestClientTest; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestClient; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.header; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +/** + * @author Michael Lavelle + */ +@RestClientTest(OpenAiTranscriptionClientWithTranscriptionResponseMetadataTests.Config.class) +public class OpenAiTranscriptionClientWithTranscriptionResponseMetadataTests { + + private static String TEST_API_KEY = "sk-1234567890"; + + @Autowired + private OpenAiAudioTranscriptionClient openAiTranscriptionClient; + + @Autowired + private MockRestServiceServer server; + + @AfterEach + void resetMockServer() { + server.reset(); + } + + @Test + void aiResponseContainsAiMetadata() { + + prepareMock(); + + Resource audioFile = new ClassPathResource("speech/jfk.flac"); + + AudioTranscriptionRequest transcriptionRequest = new AudioTranscriptionRequest(audioFile); + + AudioTranscriptionResponse response = this.openAiTranscriptionClient.call(transcriptionRequest); + + assertThat(response).isNotNull(); + + OpenAiAudioTranscriptionResponseMetadata transcriptionResponseMetadata = response.getMetadata(); + + assertThat(transcriptionResponseMetadata).isNotNull(); + + RateLimit rateLimit = transcriptionResponseMetadata.getRateLimit(); + + Duration expectedRequestsReset = Duration.ofDays(2L) + .plus(Duration.ofHours(16L)) + .plus(Duration.ofMinutes(15)) + .plus(Duration.ofSeconds(29L)); + + Duration expectedTokensReset = Duration.ofHours(27L) + .plus(Duration.ofSeconds(55L)) + .plus(Duration.ofMillis(451L)); + + assertThat(rateLimit).isNotNull(); + assertThat(rateLimit.getRequestsLimit()).isEqualTo(4000L); + assertThat(rateLimit.getRequestsRemaining()).isEqualTo(999); + assertThat(rateLimit.getRequestsReset()).isEqualTo(expectedRequestsReset); + assertThat(rateLimit.getTokensLimit()).isEqualTo(725_000L); + assertThat(rateLimit.getTokensRemaining()).isEqualTo(112_358L); + assertThat(rateLimit.getTokensReset()).isEqualTo(expectedTokensReset); + + response.getResults().forEach(transcript -> { + OpenAiAudioTranscriptionMetadata transcriptionMetadata = transcript.getMetadata(); + assertThat(transcriptionMetadata).isNotNull(); + }); + } + + private void prepareMock() { + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.set(OpenAiApiResponseHeaders.REQUESTS_LIMIT_HEADER.getName(), "4000"); + httpHeaders.set(OpenAiApiResponseHeaders.REQUESTS_REMAINING_HEADER.getName(), "999"); + httpHeaders.set(OpenAiApiResponseHeaders.REQUESTS_RESET_HEADER.getName(), "2d16h15m29s"); + httpHeaders.set(OpenAiApiResponseHeaders.TOKENS_LIMIT_HEADER.getName(), "725000"); + httpHeaders.set(OpenAiApiResponseHeaders.TOKENS_REMAINING_HEADER.getName(), "112358"); + httpHeaders.set(OpenAiApiResponseHeaders.TOKENS_RESET_HEADER.getName(), "27h55s451ms"); + + server.expect(requestTo("/v1/audio/transcriptions")) + .andExpect(method(HttpMethod.POST)) + .andExpect(header(HttpHeaders.AUTHORIZATION, "Bearer " + TEST_API_KEY)) + .andRespond(withSuccess(getJson(), MediaType.APPLICATION_JSON).headers(httpHeaders)); + + } + + private String getJson() { + return """ + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0613", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "I surrender!" + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + """; + } + + @SpringBootConfiguration + static class Config { + + @Bean + public OpenAiAudioApi chatCompletionApi(RestClient.Builder builder) { + return new OpenAiAudioApi("", TEST_API_KEY, builder); + } + + @Bean + public OpenAiAudioTranscriptionClient openAiClient(OpenAiAudioApi openAiAudioApi) { + return new OpenAiAudioTranscriptionClient(openAiAudioApi); + } + + } + +} diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/TranscriptionClientTests.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/TranscriptionClientTests.java new file mode 100644 index 000000000..104a5f0cb --- /dev/null +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/audio/transcription/TranscriptionClientTests.java @@ -0,0 +1,85 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai.audio.transcription; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import org.springframework.core.io.Resource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +/** + * Unit Tests for {@link TranscriptionClient}. + * + * @author Michael Lavelle + */ +class TranscriptionClientTests { + + @Test + void transcrbeRequestReturnsResponseCorrectly() { + + Resource mockAudioFile = Mockito.mock(Resource.class); + + OpenAiAudioTranscriptionClient mockClient = Mockito.mock(OpenAiAudioTranscriptionClient.class); + + String mockTranscription = "All your bases are belong to us"; + + // Create a mock Transcript + AudioTranscription transcript = Mockito.mock(AudioTranscription.class); + when(transcript.getOutput()).thenReturn(mockTranscription); + + // Create a mock TranscriptionResponse with the mock Transcript + AudioTranscriptionResponse response = Mockito.mock(AudioTranscriptionResponse.class); + when(response.getResult()).thenReturn(transcript); + + // Transcript transcript = spy(new Transcript(responseMessage)); + // TranscriptionResponse response = spy(new + // TranscriptionResponse(Collections.singletonList(transcript))); + + doCallRealMethod().when(mockClient).call(any(Resource.class)); + + doAnswer(invocationOnMock -> { + + AudioTranscriptionRequest transcriptionRequest = invocationOnMock.getArgument(0); + + assertThat(transcriptionRequest).isNotNull(); + assertThat(transcriptionRequest.getInstructions()).isEqualTo(mockAudioFile); + + return response; + + }).when(mockClient).call(any(AudioTranscriptionRequest.class)); + + assertThat(mockClient.call(mockAudioFile)).isEqualTo(mockTranscription); + + verify(mockClient, times(1)).call(eq(mockAudioFile)); + verify(mockClient, times(1)).call(isA(AudioTranscriptionRequest.class)); + verify(response, times(1)).getResult(); + verify(transcript, times(1)).getOutput(); + verifyNoMoreInteractions(mockClient, transcript, response); + } + +} diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java index 2e878f802..693c68edb 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java @@ -14,6 +14,7 @@ import org.springframework.ai.chat.prompt.PromptTemplate; import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.SystemMessage; import org.springframework.ai.image.ImageClient; +import org.springframework.ai.openai.audio.transcription.OpenAiAudioTranscriptionClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; @@ -28,6 +29,9 @@ public abstract class AbstractIT { @Autowired protected ChatClient openAiChatClient; + @Autowired + protected OpenAiAudioTranscriptionClient openAiTranscriptionClient; + @Autowired protected ImageClient openaiImageClient; diff --git a/models/spring-ai-openai/src/test/resources/speech/jfk.flac b/models/spring-ai-openai/src/test/resources/speech/jfk.flac new file mode 100644 index 000000000..e44b7c138 Binary files /dev/null and b/models/spring-ai-openai/src/test/resources/speech/jfk.flac differ diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAudioTranscriptionProperties.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAudioTranscriptionProperties.java new file mode 100644 index 000000000..cbdfade66 --- /dev/null +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAudioTranscriptionProperties.java @@ -0,0 +1,50 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.autoconfigure.openai; + +import org.springframework.ai.openai.api.OpenAiAudioApi; +import org.springframework.ai.openai.audio.transcription.OpenAiAudioTranscriptionOptions; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +@ConfigurationProperties(OpenAiAudioTranscriptionProperties.CONFIG_PREFIX) +public class OpenAiAudioTranscriptionProperties extends OpenAiParentProperties { + + public static final String CONFIG_PREFIX = "spring.ai.openai.audio.transcription"; + + public static final String DEFAULT_TRANSCRIPTION_MODEL = OpenAiAudioApi.WhisperModel.WHISPER_1.getValue(); + + private static final Double DEFAULT_TEMPERATURE = 0.7; + + private static final OpenAiAudioApi.TranscriptResponseFormat DEFAULT_RESPONSE_FORMAT = OpenAiAudioApi.TranscriptResponseFormat.TEXT; + + @NestedConfigurationProperty + private OpenAiAudioTranscriptionOptions options = OpenAiAudioTranscriptionOptions.builder() + .withModel(DEFAULT_TRANSCRIPTION_MODEL) + .withTemperature(DEFAULT_TEMPERATURE.floatValue()) + .withResponseFormat(DEFAULT_RESPONSE_FORMAT) + .build(); + + public OpenAiAudioTranscriptionOptions getOptions() { + return options; + } + + public void setOptions(OpenAiAudioTranscriptionOptions options) { + this.options = options; + } + +} diff --git a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfiguration.java b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfiguration.java index 2e376668d..12628318f 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfiguration.java +++ b/spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfiguration.java @@ -18,13 +18,16 @@ package org.springframework.ai.autoconfigure.openai; import java.util.List; +import org.springframework.ai.embedding.EmbeddingClient; import org.springframework.ai.model.function.FunctionCallback; import org.springframework.ai.model.function.FunctionCallbackContext; import org.springframework.ai.openai.OpenAiChatClient; import org.springframework.ai.openai.OpenAiEmbeddingClient; import org.springframework.ai.openai.OpenAiImageClient; import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.ai.openai.api.OpenAiAudioApi; import org.springframework.ai.openai.api.OpenAiImageApi; +import org.springframework.ai.openai.audio.transcription.OpenAiAudioTranscriptionClient; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -38,13 +41,13 @@ import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.springframework.web.client.RestClient; -@AutoConfiguration(after = { RestClientAutoConfiguration.class }) -@ConditionalOnClass(OpenAiApi.class) -@EnableConfigurationProperties({ OpenAiConnectionProperties.class, OpenAiChatProperties.class, - OpenAiEmbeddingProperties.class, OpenAiImageProperties.class }) /** * @author Christian Tzolov */ +@AutoConfiguration(after = { RestClientAutoConfiguration.class }) +@ConditionalOnClass(OpenAiApi.class) +@EnableConfigurationProperties({ OpenAiConnectionProperties.class, OpenAiChatProperties.class, + OpenAiEmbeddingProperties.class, OpenAiImageProperties.class, OpenAiAudioTranscriptionProperties.class }) public class OpenAiAutoConfiguration { @Bean @@ -69,7 +72,7 @@ public class OpenAiAutoConfiguration { @ConditionalOnMissingBean @ConditionalOnProperty(prefix = OpenAiEmbeddingProperties.CONFIG_PREFIX, name = "enabled", havingValue = "true", matchIfMissing = true) - public OpenAiEmbeddingClient openAiEmbeddingClient(OpenAiConnectionProperties commonProperties, + public EmbeddingClient openAiEmbeddingClient(OpenAiConnectionProperties commonProperties, OpenAiEmbeddingProperties embeddingProperties, RestClient.Builder restClientBuilder) { var openAiApi = openAiApi(embeddingProperties.getBaseUrl(), commonProperties.getBaseUrl(), @@ -111,6 +114,28 @@ public class OpenAiAutoConfiguration { return new OpenAiImageClient(openAiImageApi).withDefaultOptions(imageProperties.getOptions()); } + @Bean + @ConditionalOnMissingBean + public OpenAiAudioTranscriptionClient openAiAudioTranscriptionClient(OpenAiConnectionProperties commonProperties, + OpenAiAudioTranscriptionProperties transcriptionProperties) { + + String apiKey = StringUtils.hasText(transcriptionProperties.getApiKey()) ? transcriptionProperties.getApiKey() + : commonProperties.getApiKey(); + + String baseUrl = StringUtils.hasText(transcriptionProperties.getBaseUrl()) + ? transcriptionProperties.getBaseUrl() : commonProperties.getBaseUrl(); + + Assert.hasText(apiKey, "OpenAI API key must be set"); + Assert.hasText(baseUrl, "OpenAI base URL must be set"); + + var openAiAudioApi = new OpenAiAudioApi(baseUrl, apiKey, RestClient.builder()); + + OpenAiAudioTranscriptionClient openAiChatClient = new OpenAiAudioTranscriptionClient(openAiAudioApi, + transcriptionProperties.getOptions()); + + return openAiChatClient; + } + @Bean @ConditionalOnMissingBean public FunctionCallbackContext springAiFunctionManager(ApplicationContext context) { diff --git a/spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index e8b9f4540..f2ced6509 100644 --- a/spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -22,3 +22,4 @@ org.springframework.ai.autoconfigure.vectorstore.azure.AzureVectorStoreAutoConfi org.springframework.ai.autoconfigure.vectorstore.weaviate.WeaviateVectorStoreAutoConfiguration org.springframework.ai.autoconfigure.vectorstore.neo4j.Neo4jVectorStoreAutoConfiguration org.springframework.ai.autoconfigure.vectorstore.qdrant.QdrantVectorStoreAutoConfiguration + diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java index 1ec2d1859..ed7871ac2 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiAutoConfigurationIT.java @@ -28,6 +28,9 @@ import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.image.ImagePrompt; import org.springframework.ai.image.ImageResponse; import org.springframework.ai.openai.OpenAiImageClient; +import org.springframework.ai.openai.audio.transcription.OpenAiAudioTranscriptionClient; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; import reactor.core.publisher.Flux; import org.springframework.ai.chat.ChatResponse; @@ -59,6 +62,17 @@ public class OpenAiAutoConfigurationIT { }); } + @Test + void transcribe() { + contextRunner.run(context -> { + OpenAiAudioTranscriptionClient client = context.getBean(OpenAiAudioTranscriptionClient.class); + Resource audioFile = new ClassPathResource("/speech/jfk.flac"); + String response = client.call(audioFile); + assertThat(response).isNotEmpty(); + logger.info("Response: " + response); + }); + } + @Test void generateStreaming() { contextRunner.run(context -> { diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiPropertiesTests.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiPropertiesTests.java index 5c3237a12..aeeb7ae89 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiPropertiesTests.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiPropertiesTests.java @@ -26,6 +26,7 @@ import org.springframework.ai.openai.OpenAiImageClient; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ResponseFormat; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ToolChoiceBuilder; import org.springframework.ai.openai.api.OpenAiApi.FunctionTool.Type; +import org.springframework.ai.openai.api.OpenAiAudioApi; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.autoconfigure.web.client.RestClientAutoConfiguration; import org.springframework.boot.test.context.runner.ApplicationContextRunner; @@ -68,6 +69,32 @@ public class OpenAiPropertiesTests { }); } + @Test + public void transcriptionProperties() { + + new ApplicationContextRunner().withPropertyValues( + // @formatter:off + "spring.ai.openai.base-url=TEST_BASE_URL", + "spring.ai.openai.api-key=abc123", + "spring.ai.openai.audio.transcription.options.model=MODEL_XYZ", + "spring.ai.openai.audio.transcription.options.temperature=0.55") + // @formatter:on + .withConfiguration(AutoConfigurations.of(RestClientAutoConfiguration.class, OpenAiAutoConfiguration.class)) + .run(context -> { + var transcriptionProperties = context.getBean(OpenAiAudioTranscriptionProperties.class); + var connectionProperties = context.getBean(OpenAiConnectionProperties.class); + + assertThat(connectionProperties.getApiKey()).isEqualTo("abc123"); + assertThat(connectionProperties.getBaseUrl()).isEqualTo("TEST_BASE_URL"); + + assertThat(transcriptionProperties.getApiKey()).isNull(); + assertThat(transcriptionProperties.getBaseUrl()).isNull(); + + assertThat(transcriptionProperties.getOptions().getModel()).isEqualTo("MODEL_XYZ"); + assertThat(transcriptionProperties.getOptions().getTemperature()).isEqualTo(0.55f); + }); + } + @Test public void chatOverrideConnectionProperties() { @@ -96,6 +123,34 @@ public class OpenAiPropertiesTests { }); } + @Test + public void transcriptionOverrideConnectionProperties() { + + new ApplicationContextRunner().withPropertyValues( + // @formatter:off + "spring.ai.openai.base-url=TEST_BASE_URL", + "spring.ai.openai.api-key=abc123", + "spring.ai.openai.audio.transcription.base-url=TEST_BASE_URL2", + "spring.ai.openai.audio.transcription.api-key=456", + "spring.ai.openai.audio.transcription.options.model=MODEL_XYZ", + "spring.ai.openai.audio.transcription.options.temperature=0.55") + // @formatter:on + .withConfiguration(AutoConfigurations.of(RestClientAutoConfiguration.class, OpenAiAutoConfiguration.class)) + .run(context -> { + var transcriptionProperties = context.getBean(OpenAiAudioTranscriptionProperties.class); + var connectionProperties = context.getBean(OpenAiConnectionProperties.class); + + assertThat(connectionProperties.getApiKey()).isEqualTo("abc123"); + assertThat(connectionProperties.getBaseUrl()).isEqualTo("TEST_BASE_URL"); + + assertThat(transcriptionProperties.getApiKey()).isEqualTo("456"); + assertThat(transcriptionProperties.getBaseUrl()).isEqualTo("TEST_BASE_URL2"); + + assertThat(transcriptionProperties.getOptions().getModel()).isEqualTo("MODEL_XYZ"); + assertThat(transcriptionProperties.getOptions().getTemperature()).isEqualTo(0.55f); + }); + } + @Test public void embeddingProperties() { @@ -288,6 +343,41 @@ public class OpenAiPropertiesTests { }); } + @Test + public void transcriptionOptionsTest() { + + new ApplicationContextRunner().withPropertyValues( + // @formatter:off + "spring.ai.openai.api-key=API_KEY", + "spring.ai.openai.base-url=TEST_BASE_URL", + + "spring.ai.openai.audio.transcription.options.model=MODEL_XYZ", + "spring.ai.openai.audio.transcription.options.language=en", + "spring.ai.openai.audio.transcription.options.prompt=Er, yes, I think so", + "spring.ai.openai.audio.transcription.options.responseFormat=JSON", + "spring.ai.openai.audio.transcription.options.temperature=0.55" + ) + // @formatter:on + .withConfiguration(AutoConfigurations.of(RestClientAutoConfiguration.class, OpenAiAutoConfiguration.class)) + .run(context -> { + var transcriptionProperties = context.getBean(OpenAiAudioTranscriptionProperties.class); + var connectionProperties = context.getBean(OpenAiConnectionProperties.class); + var embeddingProperties = context.getBean(OpenAiEmbeddingProperties.class); + + assertThat(connectionProperties.getBaseUrl()).isEqualTo("TEST_BASE_URL"); + assertThat(connectionProperties.getApiKey()).isEqualTo("API_KEY"); + + assertThat(embeddingProperties.getOptions().getModel()).isEqualTo("text-embedding-ada-002"); + + assertThat(transcriptionProperties.getOptions().getModel()).isEqualTo("MODEL_XYZ"); + assertThat(transcriptionProperties.getOptions().getLanguage()).isEqualTo("en"); + assertThat(transcriptionProperties.getOptions().getPrompt()).isEqualTo("Er, yes, I think so"); + assertThat(transcriptionProperties.getOptions().getResponseFormat()) + .isEqualTo(OpenAiAudioApi.TranscriptResponseFormat.JSON); + assertThat(transcriptionProperties.getOptions().getTemperature()).isEqualTo(0.55f); + }); + } + @Test public void embeddingOptionsTest() { diff --git a/spring-ai-spring-boot-autoconfigure/src/test/resources/speech/jfk.flac b/spring-ai-spring-boot-autoconfigure/src/test/resources/speech/jfk.flac new file mode 100644 index 000000000..e44b7c138 Binary files /dev/null and b/spring-ai-spring-boot-autoconfigure/src/test/resources/speech/jfk.flac differ