GH-3118: Fix NPE in OpenAIChatModel compatible streaming api with vertex ai gemini

Fixes: #3118

Signed-off-by: jonghoon park <dev@jonghoonpark.com>
This commit is contained in:
jonghoon park
2025-05-13 02:28:00 +09:00
committed by Soby Chacko
parent 2d517eec5c
commit 0c95cc396f
2 changed files with 125 additions and 3 deletions

View File

@@ -104,6 +104,7 @@ import org.springframework.util.StringUtils;
* @author Ilayaperumal Gopinathan
* @author Alexandros Pappas
* @author Soby Chacko
* @author Jonghoon Park
* @see ChatModel
* @see StreamingChatModel
* @see OpenAiApi
@@ -304,15 +305,15 @@ public class OpenAiChatModel implements ChatModel {
Flux<ChatResponse> chatResponse = completionChunks.map(this::chunkToChatCompletion)
.switchMap(chatCompletion -> Mono.just(chatCompletion).map(chatCompletion2 -> {
try {
@SuppressWarnings("null")
String id = chatCompletion2.id();
// If an id is not provided, set to "NO_ID" (for compatible APIs).
String id = chatCompletion2.id() == null ? "NO_ID" : chatCompletion2.id();
List<Generation> generations = chatCompletion2.choices().stream().map(choice -> { // @formatter:off
if (choice.message().role() != null) {
roleMap.putIfAbsent(id, choice.message().role().name());
}
Map<String, Object> metadata = Map.of(
"id", chatCompletion2.id(),
"id", id,
"role", roleMap.getOrDefault(id, ""),
"index", choice.index(),
"finishReason", choice.finishReason() != null ? choice.finishReason().name() : "",

View File

@@ -0,0 +1,121 @@
/*
* Copyright 2025-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.openai.chat.proxy;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.chat.prompt.SystemPromptTemplate;
import org.springframework.ai.model.SimpleApiKey;
import org.springframework.ai.model.tool.ToolCallingManager;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.Resource;
import reactor.core.publisher.Flux;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Jonghoon Park
*/
@SpringBootTest(classes = VertexAIGeminiWithOpenAiChatModelIT.Config.class)
@EnabledIfEnvironmentVariable(named = "GEMINI_API_KEY", matches = ".+")
class VertexAIGeminiWithOpenAiChatModelIT {
private static final String VERTEX_AI_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com";
private static final String VERTEX_AI_GEMINI_DEFAULT_MODEL = "gemini-2.0-flash";
@Value("classpath:/prompts/system-message.st")
private Resource systemResource;
@Autowired
private OpenAiChatModel chatModel;
@Test
void roleTest() {
UserMessage userMessage = new UserMessage(
"Tell me about 3 famous pirates from the Golden Age of Piracy and what they did.");
SystemPromptTemplate systemPromptTemplate = new SystemPromptTemplate(this.systemResource);
Message systemMessage = systemPromptTemplate.createMessage(Map.of("name", "Bob", "voice", "pirate"));
Prompt prompt = new Prompt(List.of(userMessage, systemMessage));
ChatResponse response = this.chatModel.call(prompt);
assertThat(response.getResults()).hasSize(1);
assertThat(response.getResults().get(0).getOutput().getText()).contains("Blackbeard");
}
@Test
void streamRoleTest() {
UserMessage userMessage = new UserMessage(
"Tell me about 3 famous pirates from the Golden Age of Piracy and what they did.");
SystemPromptTemplate systemPromptTemplate = new SystemPromptTemplate(this.systemResource);
Message systemMessage = systemPromptTemplate.createMessage(Map.of("name", "Bob", "voice", "pirate"));
Prompt prompt = new Prompt(List.of(userMessage, systemMessage));
Flux<ChatResponse> flux = this.chatModel.stream(prompt);
List<ChatResponse> responses = flux.collectList().block();
assertThat(responses.size()).isGreaterThan(1);
String stitchedResponseContent = responses.stream()
.map(ChatResponse::getResults)
.flatMap(List::stream)
.map(Generation::getOutput)
.map(AssistantMessage::getText)
.collect(Collectors.joining());
assertThat(stitchedResponseContent).contains("Blackbeard");
}
@SpringBootConfiguration
static class Config {
@Bean
public OpenAiApi chatCompletionApi() {
return OpenAiApi.builder()
.baseUrl(VERTEX_AI_GEMINI_BASE_URL)
.completionsPath("/v1beta/openai/chat/completions")
.apiKey(new SimpleApiKey(System.getenv("GEMINI_API_KEY")))
.build();
}
@Bean
public OpenAiChatModel openAiClient(OpenAiApi openAiApi) {
return OpenAiChatModel.builder()
.openAiApi(openAiApi)
.toolCallingManager(ToolCallingManager.builder().build())
.defaultOptions(OpenAiChatOptions.builder().model(VERTEX_AI_GEMINI_DEFAULT_MODEL).build())
.build();
}
}
}