diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java index 01f619b0a..05583f0b1 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java @@ -217,7 +217,8 @@ public class OpenAiChatModel implements ChatModel { "role", choice.message().role() != null ? choice.message().role().name() : "", "index", choice.index(), "finishReason", choice.finishReason() != null ? choice.finishReason().name() : "", - "refusal", StringUtils.hasText(choice.message().refusal()) ? choice.message().refusal() : ""); + "refusal", StringUtils.hasText(choice.message().refusal()) ? choice.message().refusal() : "", + "annotations", choice.message().annotations() != null ? choice.message().annotations() : List.of()); return buildGeneration(choice, metadata, request); }).toList(); // @formatter:on @@ -315,8 +316,8 @@ public class OpenAiChatModel implements ChatModel { "role", roleMap.getOrDefault(id, ""), "index", choice.index(), "finishReason", choice.finishReason() != null ? choice.finishReason().name() : "", - "refusal", StringUtils.hasText(choice.message().refusal()) ? choice.message().refusal() : ""); - + "refusal", StringUtils.hasText(choice.message().refusal()) ? choice.message().refusal() : "", + "annotations", choice.message().annotations() != null ? choice.message().annotations() : List.of()); return buildGeneration(choice, metadata, request); }).toList(); // @formatter:on @@ -583,7 +584,7 @@ public class OpenAiChatModel implements ChatModel { } return List.of(new ChatCompletionMessage(assistantMessage.getText(), - ChatCompletionMessage.Role.ASSISTANT, null, null, toolCalls, null, audioOutput)); + ChatCompletionMessage.Role.ASSISTANT, null, null, toolCalls, null, audioOutput, null)); } else if (message.getMessageType() == MessageType.TOOL) { ToolResponseMessage toolMessage = (ToolResponseMessage) message; @@ -593,7 +594,7 @@ public class OpenAiChatModel implements ChatModel { return toolMessage.getResponses() .stream() .map(tr -> new ChatCompletionMessage(tr.responseData(), ChatCompletionMessage.Role.TOOL, tr.name(), - tr.id(), null, null, null)) + tr.id(), null, null, null, null)) .toList(); } else { diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java index 9439c0412..a1a9fede7 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatOptions.java @@ -36,6 +36,7 @@ import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.AudioParameters; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.StreamOptions; import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ToolChoiceBuilder; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.WebSearchOptions; import org.springframework.ai.openai.api.ResponseFormat; import org.springframework.ai.tool.ToolCallback; import org.springframework.lang.Nullable; @@ -194,6 +195,11 @@ public class OpenAiChatOptions implements ToolCallingChatOptions { */ private @JsonProperty("reasoning_effort") String reasoningEffort; + /** + * This tool searches the web for relevant results to use in a response. + */ + private @JsonProperty("web_search_options") WebSearchOptions webSearchOptions; + /** * Collection of {@link ToolCallback}s to be used for tool calling in the chat completion requests. */ @@ -548,6 +554,14 @@ public class OpenAiChatOptions implements ToolCallingChatOptions { this.reasoningEffort = reasoningEffort; } + public WebSearchOptions getWebSearchOptions() { + return this.webSearchOptions; + } + + public void setWebSearchOptions(WebSearchOptions webSearchOptions) { + this.webSearchOptions = webSearchOptions; + } + @Override public OpenAiChatOptions copy() { return OpenAiChatOptions.fromOptions(this); @@ -560,7 +574,7 @@ public class OpenAiChatOptions implements ToolCallingChatOptions { this.streamOptions, this.seed, this.stop, this.temperature, this.topP, this.tools, this.toolChoice, this.user, this.parallelToolCalls, this.toolCallbacks, this.toolNames, this.httpHeaders, this.internalToolExecutionEnabled, this.toolContext, this.outputModalities, this.outputAudio, - this.store, this.metadata, this.reasoningEffort); + this.store, this.metadata, this.reasoningEffort, this.webSearchOptions); } @Override @@ -592,7 +606,8 @@ public class OpenAiChatOptions implements ToolCallingChatOptions { && Objects.equals(this.outputModalities, other.outputModalities) && Objects.equals(this.outputAudio, other.outputAudio) && Objects.equals(this.store, other.store) && Objects.equals(this.metadata, other.metadata) - && Objects.equals(this.reasoningEffort, other.reasoningEffort); + && Objects.equals(this.reasoningEffort, other.reasoningEffort) + && Objects.equals(this.webSearchOptions, other.webSearchOptions); } @Override @@ -780,6 +795,11 @@ public class OpenAiChatOptions implements ToolCallingChatOptions { return this; } + public Builder webSearchOptions(WebSearchOptions webSearchOptions) { + this.options.webSearchOptions = webSearchOptions; + return this; + } + public OpenAiChatOptions build() { return this.options; } diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java index a0da1221d..d31fad869 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java @@ -590,7 +590,21 @@ public class OpenAiApi { * Context window: 4,096 tokens. Max output tokens: 4,096 tokens. Knowledge * cutoff: September, 2021. */ - GPT_3_5_TURBO_INSTRUCT("gpt-3.5-turbo-instruct"); + GPT_3_5_TURBO_INSTRUCT("gpt-3.5-turbo-instruct"), + + /** + * GPT-4o Search Preview is a specialized model for web search in Chat + * Completions. It is trained to understand and execute web search queries. See + * the web search guide for more information. + */ + GPT_4_O_SEARCH_PREVIEW("gpt-4o-search-preview"), + + /** + * GPT-4o mini Search Preview is a specialized model for web search in Chat + * Completions. It is trained to understand and execute web search queries. See + * the web search guide for more information. + */ + GPT_4_O_MINI_SEARCH_PREVIEW("gpt-4o-mini-search-preview"); public final String value; @@ -951,6 +965,10 @@ public class OpenAiApi { * @param parallelToolCalls If set to true, the model will call all functions in the * tools list in parallel. Otherwise, the model will call the functions in the tools * list in the order they are provided. + * @param reasoningEffort Constrains effort on reasoning for reasoning models. + * Currently supported values are low, medium, and high. Reducing reasoning effort can + * result in faster responses and fewer tokens used on reasoning in a response. + * @param webSearchOptions Options for web search. */ @JsonInclude(Include.NON_NULL) public record ChatCompletionRequest(// @formatter:off @@ -980,7 +998,8 @@ public class OpenAiApi { @JsonProperty("tool_choice") Object toolChoice, @JsonProperty("parallel_tool_calls") Boolean parallelToolCalls, @JsonProperty("user") String user, - @JsonProperty("reasoning_effort") String reasoningEffort) { + @JsonProperty("reasoning_effort") String reasoningEffort, + @JsonProperty("web_search_options") WebSearchOptions webSearchOptions) { /** * Shortcut constructor for a chat completion request with the given messages, model and temperature. @@ -992,7 +1011,7 @@ public class OpenAiApi { public ChatCompletionRequest(List messages, String model, Double temperature) { this(messages, model, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, false, null, temperature, null, - null, null, null, null, null); + null, null, null, null, null, null); } /** @@ -1006,7 +1025,7 @@ public class OpenAiApi { this(messages, model, null, null, null, null, null, null, null, null, null, List.of(OutputModality.AUDIO, OutputModality.TEXT), audio, null, null, null, null, null, stream, null, null, null, - null, null, null, null, null); + null, null, null, null, null, null); } /** @@ -1021,7 +1040,7 @@ public class OpenAiApi { public ChatCompletionRequest(List messages, String model, Double temperature, boolean stream) { this(messages, model, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, stream, null, temperature, null, - null, null, null, null, null); + null, null, null, null, null, null); } /** @@ -1037,7 +1056,7 @@ public class OpenAiApi { List tools, Object toolChoice) { this(messages, model, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, false, null, 0.8, null, - tools, toolChoice, null, null, null); + tools, toolChoice, null, null, null, null); } /** @@ -1050,7 +1069,7 @@ public class OpenAiApi { public ChatCompletionRequest(List messages, Boolean stream) { this(messages, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, stream, null, null, null, - null, null, null, null, null); + null, null, null, null, null, null); } /** @@ -1063,7 +1082,7 @@ public class OpenAiApi { return new ChatCompletionRequest(this.messages, this.model, this.store, this.metadata, this.frequencyPenalty, this.logitBias, this.logprobs, this.topLogprobs, this.maxTokens, this.maxCompletionTokens, this.n, this.outputModalities, this.audioParameters, this.presencePenalty, this.responseFormat, this.seed, this.serviceTier, this.stop, this.stream, streamOptions, this.temperature, this.topP, - this.tools, this.toolChoice, this.parallelToolCalls, this.user, this.reasoningEffort); + this.tools, this.toolChoice, this.parallelToolCalls, this.user, this.reasoningEffort, this.webSearchOptions); } /** @@ -1145,6 +1164,61 @@ public class OpenAiApi { public static StreamOptions INCLUDE_USAGE = new StreamOptions(true); } + + /** + * This tool searches the web for relevant results to use in a response. + * + * @param searchContextSize + * @param userLocation + */ + @JsonInclude(Include.NON_NULL) + public record WebSearchOptions(@JsonProperty("search_context_size") SearchContextSize searchContextSize, + @JsonProperty("user_location") UserLocation userLocation) { + + /** + * High level guidance for the amount of context window space to use for the + * search. One of low, medium, or high. medium is the default. + */ + public enum SearchContextSize { + + /** + * Low context size. + */ + @JsonProperty("low") + LOW, + + /** + * Medium context size. This is the default. + */ + @JsonProperty("medium") + MEDIUM, + + /** + * High context size. + */ + @JsonProperty("high") + HIGH + + } + + /** + * Approximate location parameters for the search. + * + * @param type The type of location approximation. Always "approximate". + * @param approximate The approximate location details. + */ + @JsonInclude(Include.NON_NULL) + public record UserLocation(@JsonProperty("type") String type, + @JsonProperty("approximate") Approximate approximate) { + + @JsonInclude(Include.NON_NULL) + public record Approximate(@JsonProperty("city") String city, @JsonProperty("country") String country, + @JsonProperty("region") String region, @JsonProperty("timezone") String timezone) { + } + } + + } + } // @formatter:on /** @@ -1163,19 +1237,22 @@ public class OpenAiApi { * Applicable only for {@link Role#ASSISTANT} role and null otherwise. * @param refusal The refusal message by the assistant. Applicable only for * {@link Role#ASSISTANT} role and null otherwise. - * @param audioOutput Audio response from the model. >>>>>>> bdb66e577 (OpenAI - - * Support audio input modality) + * @param audioOutput Audio response from the model. + * @param annotations Annotations for the message, when applicable, as when using the + * web search tool. */ - @JsonInclude(Include.NON_NULL) - public record ChatCompletionMessage(// @formatter:off + @JsonInclude(JsonInclude.Include.NON_NULL) + public record ChatCompletionMessage( + // @formatter:off @JsonProperty("content") Object rawContent, @JsonProperty("role") Role role, @JsonProperty("name") String name, @JsonProperty("tool_call_id") String toolCallId, - @JsonProperty("tool_calls") - @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY) List toolCalls, + @JsonProperty("tool_calls") @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY) List toolCalls, @JsonProperty("refusal") String refusal, - @JsonProperty("audio") AudioOutput audioOutput) { // @formatter:on + @JsonProperty("audio") AudioOutput audioOutput, + @JsonProperty("annotations") List annotations + ) { // @formatter:on /** * Create a chat completion message with the given content and role. All other @@ -1184,8 +1261,7 @@ public class OpenAiApi { * @param role The role of the author of this message. */ public ChatCompletionMessage(Object content, Role role) { - this(content, role, null, null, null, null, null); - + this(content, role, null, null, null, null, null, null); } /** @@ -1362,6 +1438,29 @@ public class OpenAiApi { @JsonProperty("transcript") String transcript ) { // @formatter:on } + + /** + * Represents an annotation within a message, specifically for URL citations. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + public record Annotation(@JsonProperty("type") String type, + @JsonProperty("url_citation") UrlCitation urlCitation) { + /** + * A URL citation when using web search. + * + * @param endIndex The index of the last character of the URL citation in the + * message. + * @param startIndex The index of the first character of the URL citation in + * the message. + * @param title The title of the web resource. + * @param url The URL of the web resource. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + public record UrlCitation(@JsonProperty("end_index") Integer endIndex, + @JsonProperty("start_index") Integer startIndex, @JsonProperty("title") String title, + @JsonProperty("url") String url) { + } + } } /** diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiStreamFunctionCallingHelper.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiStreamFunctionCallingHelper.java index 17be23c4b..e159d0362 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiStreamFunctionCallingHelper.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiStreamFunctionCallingHelper.java @@ -40,6 +40,7 @@ import org.springframework.util.StringUtils; * * @author Christian Tzolov * @author Thomas Vitale + * @author Alexandros Pappas * @since 0.8.1 */ public class OpenAiStreamFunctionCallingHelper { @@ -98,6 +99,8 @@ public class OpenAiStreamFunctionCallingHelper { String refusal = (current.refusal() != null ? current.refusal() : previous.refusal()); ChatCompletionMessage.AudioOutput audioOutput = (current.audioOutput() != null ? current.audioOutput() : previous.audioOutput()); + List annotations = (current.annotations() != null ? current.annotations() + : previous.annotations()); List toolCalls = new ArrayList<>(); ToolCall lastPreviousTooCall = null; @@ -127,7 +130,7 @@ public class OpenAiStreamFunctionCallingHelper { toolCalls.add(lastPreviousTooCall); } } - return new ChatCompletionMessage(content, role, name, toolCallId, toolCalls, refusal, audioOutput); + return new ChatCompletionMessage(content, role, name, toolCallId, toolCalls, refusal, audioOutput, annotations); } private ToolCall merge(ToolCall previous, ToolCall current) { diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java index f843c0c73..36c1eb84a 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/OpenAiApiIT.java @@ -75,7 +75,7 @@ public class OpenAiApiIT { "If a train travels 100 miles in 2 hours, what is its average speed?", ChatCompletionMessage.Role.USER); ChatCompletionRequest request = new ChatCompletionRequest(List.of(userMessage), "o1", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, false, null, null, null, null, - null, null, null, "low"); + null, null, null, "low", null); ResponseEntity response = this.openAiApi.chatCompletionEntity(request); assertThat(response).isNotNull(); diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/tool/OpenAiApiToolFunctionCallIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/tool/OpenAiApiToolFunctionCallIT.java index 482fc1157..e655de464 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/tool/OpenAiApiToolFunctionCallIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/api/tool/OpenAiApiToolFunctionCallIT.java @@ -44,6 +44,7 @@ import static org.assertj.core.api.Assertions.assertThat; * * @author Christian Tzolov * @author Thomas Vitale + * @author Alexandros Pappas */ @EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+") public class OpenAiApiToolFunctionCallIT { @@ -129,7 +130,7 @@ public class OpenAiApiToolFunctionCallIT { // extend conversation with function response. messages.add(new ChatCompletionMessage("" + weatherResponse.temp() + weatherRequest.unit(), Role.TOOL, - functionName, toolCall.id(), null, null, null)); + functionName, toolCall.id(), null, null, null, null)); } } diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java index 310f2202d..747f7d7ca 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java @@ -709,6 +709,72 @@ public class OpenAiChatModelIT extends AbstractIT { } + @Test + void webSearchAnnotationsTest() { + UserMessage userMessage = new UserMessage("What is the latest news on the Mars rover?"); + + var promptOptions = OpenAiChatOptions.builder() + .model(OpenAiApi.ChatModel.GPT_4_O_SEARCH_PREVIEW.getValue()) + .webSearchOptions(new OpenAiApi.ChatCompletionRequest.WebSearchOptions( + OpenAiApi.ChatCompletionRequest.WebSearchOptions.SearchContextSize.MEDIUM, + new OpenAiApi.ChatCompletionRequest.WebSearchOptions.UserLocation("approximate", + new OpenAiApi.ChatCompletionRequest.WebSearchOptions.UserLocation.Approximate( + "San Francisco", "US", "California", "America/Los_Angeles")))) + .build(); + + ChatResponse response = this.chatModel.call(new Prompt(List.of(userMessage), promptOptions)); + + logger.info("Response: {}", response); + + assertThat(response.getResult().getOutput().getText()).isNotEmpty(); + + Object annotationsRaw = response.getResult().getOutput().getMetadata().get("annotations"); + assertThat(annotationsRaw).isNotNull().isInstanceOf(List.class); + + List annotations = (List) annotationsRaw; + assertThat(annotations).isNotEmpty(); + assertThat(annotations.get(0).type()).isEqualTo("url_citation"); + assertThat(annotations.get(0).urlCitation()).isNotNull(); + assertThat(annotations.get(0).urlCitation().url()).isNotEmpty(); + } + + @Test + void streamWebSearchAnnotationsTest() { + UserMessage userMessage = new UserMessage("What is the weather in San Francisco?"); + + var promptOptions = OpenAiChatOptions.builder() + .model(OpenAiApi.ChatModel.GPT_4_O_SEARCH_PREVIEW.getValue()) + .build(); + + Flux responseFlux = this.streamingChatModel + .stream(new Prompt(List.of(userMessage), promptOptions)); + + // Collect all streamed ChatResponses into a list. + List responses = responseFlux.collectList().block(); + assert responses != null; + assertThat(responses).isNotEmpty(); + ChatResponse lastResponse = responses.get(responses.size() - 1); + logger.info("Last Response: {}", lastResponse); + + Object annotationsRaw = lastResponse.getResult().getOutput().getMetadata().get("annotations"); + assertThat(annotationsRaw).isNotNull().isInstanceOf(List.class); + + List annotations = (List) annotationsRaw; + assertThat(annotations).isNotEmpty(); + assertThat(annotations.get(0).type()).isEqualTo("url_citation"); + assertThat(annotations.get(0).urlCitation()).isNotNull(); + assertThat(annotations.get(0).urlCitation().url()).isNotEmpty(); + + // For debugging, log fullContent + String fullContent = responses.stream() + .map(ChatResponse::getResults) + .flatMap(List::stream) + .map(Generation::getOutput) + .map(AssistantMessage::getText) + .collect(Collectors.joining()); + logger.info("Full Content: {}", fullContent); + } + record ActorsFilmsRecord(String actor, List movies) { }