From dbccfda7c57b54f6199cbfb8fb55eac8916983de Mon Sep 17 00:00:00 2001 From: Christian Tzolov Date: Thu, 13 Jun 2024 12:43:18 +0200 Subject: [PATCH] Update Google VertexAI Gemini to 26.41.0 - update the Gemini function calling ITs to include a system message with dedicated calling instructions. - fix a type with few Gemini ChatModel enum names. - re-enable all Gemini ITs. --- .../gemini/VertexAiGeminiChatModel.java | 4 +- ...texAiGeminiChatModelFunctionCallingIT.java | 79 ++++++++----------- .../VertexAiGeminiPaymentTransactionIT.java | 19 ++--- pom.xml | 2 +- .../tool/FunctionCallWithFunctionBeanIT.java | 8 +- .../FunctionCallWithFunctionWrapperIT.java | 2 +- .../FunctionCallWithPromptFunctionIT.java | 2 +- 7 files changed, 50 insertions(+), 66 deletions(-) diff --git a/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModel.java b/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModel.java index 9703c9b8c..23269d7b7 100644 --- a/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModel.java +++ b/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModel.java @@ -102,9 +102,9 @@ public class VertexAiGeminiChatModel GEMINI_PRO("gemini-pro"), - GEMINI_PRO_1_5_PRO("gemini-1.5-pro-preview-0514"), + GEMINI_1_5_PRO("gemini-1.5-pro-preview-0514"), - GEMINI_PRO_1_5_FLASH("gemini-1.5-flash-preview-0514"); + GEMINI_1_5_FLASH("gemini-1.5-flash-preview-0514"); ChatModel(String value) { this.value = value; diff --git a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/function/VertexAiGeminiChatModelFunctionCallingIT.java b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/function/VertexAiGeminiChatModelFunctionCallingIT.java index 39ea0bc0f..963c0d46d 100644 --- a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/function/VertexAiGeminiChatModelFunctionCallingIT.java +++ b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/function/VertexAiGeminiChatModelFunctionCallingIT.java @@ -15,37 +15,37 @@ */ package org.springframework.ai.vertexai.gemini.function; +import static org.assertj.core.api.Assertions.assertThat; + import java.util.ArrayList; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; -import com.google.cloud.vertexai.Transport; -import com.google.cloud.vertexai.VertexAI; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.ai.vertexai.gemini.VertexAiGeminiChatModel; -import reactor.core.publisher.Flux; - -import org.springframework.ai.chat.model.ChatResponse; -import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.model.function.FunctionCallbackWrapper; import org.springframework.ai.model.function.FunctionCallbackWrapper.Builder.SchemaType; +import org.springframework.ai.vertexai.gemini.VertexAiGeminiChatModel; import org.springframework.ai.vertexai.gemini.VertexAiGeminiChatOptions; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.Bean; -import static org.assertj.core.api.Assertions.assertThat; +import com.google.cloud.vertexai.Transport; +import com.google.cloud.vertexai.VertexAI; + +import reactor.core.publisher.Flux; @SpringBootTest @EnabledIfEnvironmentVariable(named = "VERTEX_AI_GEMINI_PROJECT_ID", matches = ".*") @@ -57,27 +57,19 @@ public class VertexAiGeminiChatModelFunctionCallingIT { @Autowired private VertexAiGeminiChatModel chatModel; - @AfterEach - public void afterEach() { - try { - Thread.sleep(3000); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - } - @Test - @Disabled("Google Vertex AI degraded support for parallel function calls") public void functionCallExplicitOpenApiSchema() { - UserMessage userMessage = new UserMessage( - "What's the weather like in San Francisco, in Paris and in Tokyo, Japan?" - + " Use Celsius units. Answer for all requested locations."); - // " Use Celsius units. Use Multi-turn function calling. Provide answer for all - // requested locations."); + var systemMessage = new SystemMessage(""" + Use Multi-turn function calling. + Answer for all listed locations. + If the information was not fetched call the function again. Repeat at most 3 times. + """); - List messages = new ArrayList<>(List.of(userMessage)); + UserMessage userMessage = new UserMessage( + "What's the weather like in San Francisco, Paris and in Tokyo? Perform multiple funciton execution if necessary. Return the temperature in Celsius."); + + List messages = new ArrayList<>(List.of(systemMessage, userMessage)); String openApiSchema = """ { @@ -98,7 +90,7 @@ public class VertexAiGeminiChatModelFunctionCallingIT { """; var promptOptions = VertexAiGeminiChatOptions.builder() - .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_PRO) + .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_1_5_FLASH) // .withModel(VertexAiGeminiModelCall.ChatModel.GEMINI_PRO_1_5_PRO) .withFunctionCallbacks(List.of(FunctionCallbackWrapper.builder(new MockWeatherService()) .withName("get_current_weather") @@ -111,11 +103,7 @@ public class VertexAiGeminiChatModelFunctionCallingIT { logger.info("Response: {}", response); - // System.out.println(response.getResult().getOutput().getContent()); - assertThat(response.getResult().getOutput().getContent()).containsAnyOf("30.0", "30"); - assertThat(response.getResult().getOutput().getContent()).containsAnyOf("10.0", "10"); - assertThat(response.getResult().getOutput().getContent()).containsAnyOf("15.0", "15"); - + assertThat(response.getResult().getOutput().getContent()).contains("30", "10", "15"); } @Test @@ -126,8 +114,8 @@ public class VertexAiGeminiChatModelFunctionCallingIT { List messages = new ArrayList<>(List.of(userMessage)); var promptOptions = VertexAiGeminiChatOptions.builder() - // .withModel(VertexAiGeminiModelCall.ChatModel.GEMINI_PRO_1_5_PRO) - .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_PRO.getValue()) + .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_1_5_PRO) + // .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_PRO) .withFunctionCallbacks(List.of( FunctionCallbackWrapper.builder(new MockWeatherService()) .withSchemaType(SchemaType.OPEN_API_SCHEMA) @@ -160,15 +148,18 @@ public class VertexAiGeminiChatModelFunctionCallingIT { @Test public void functionCallTestInferredOpenApiSchemaStream() { - UserMessage userMessage = new UserMessage("What's the weather like in San Francisco in Celsius units?"); - // UserMessage userMessage = new UserMessage( - // "What's the weather like in San Francisco, in Paris and in Tokyo, Japan? Use - // Multi-turn function calling. Provide answer for all requested locations."); + var systemMessage = new SystemMessage(""" + Use Multi-turn function calling. + Answer for all listed locations. + If the information was not fetched call the function again. Repeat at most 3 times. + """); + UserMessage userMessage = new UserMessage( + "What's the weather like in San Francisco, Paris and in Tokyo? Perform multiple funciton execution if necessary. Return the temperature in Celsius."); - List messages = new ArrayList<>(List.of(userMessage)); + List messages = new ArrayList<>(List.of(systemMessage, userMessage)); var promptOptions = VertexAiGeminiChatOptions.builder() - .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_PRO) + .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_1_5_FLASH) .withFunctionCallbacks(List.of(FunctionCallbackWrapper.builder(new MockWeatherService()) .withSchemaType(SchemaType.OPEN_API_SCHEMA) .withName("getCurrentWeather") @@ -189,9 +180,7 @@ public class VertexAiGeminiChatModelFunctionCallingIT { logger.info("Response: {}", responseString); - // assertThat(responseString).containsAnyOf("15.0", "15"); - assertThat(responseString).containsAnyOf("30.0", "30"); - // assertThat(responseString).containsAnyOf("10.0", "10"); + assertThat(responseString).contains("30", "10", "15"); } @@ -227,7 +216,7 @@ public class VertexAiGeminiChatModelFunctionCallingIT { public VertexAiGeminiChatModel vertexAiEmbedding(VertexAI vertexAi) { return new VertexAiGeminiChatModel(vertexAi, VertexAiGeminiChatOptions.builder() - .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_PRO) + .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_1_5_PRO) .withTemperature(0.9f) .build()); } diff --git a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/function/VertexAiGeminiPaymentTransactionIT.java b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/function/VertexAiGeminiPaymentTransactionIT.java index dd0f447a8..348809008 100644 --- a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/function/VertexAiGeminiPaymentTransactionIT.java +++ b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/function/VertexAiGeminiPaymentTransactionIT.java @@ -21,16 +21,11 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; -import com.google.cloud.vertexai.Transport; -import com.google.cloud.vertexai.VertexAI; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import reactor.core.publisher.Flux; - import org.springframework.ai.chat.client.AdvisedRequest; import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.chat.client.RequestResponseAdvisor; @@ -46,10 +41,14 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Description; +import com.google.cloud.vertexai.Transport; +import com.google.cloud.vertexai.VertexAI; + +import reactor.core.publisher.Flux; + /** * @author Christian Tzolov */ -@Disabled("Vertex AI Gemini function calling is very unstable.") @SpringBootTest @EnabledIfEnvironmentVariable(named = "VERTEX_AI_GEMINI_PROJECT_ID", matches = ".*") @EnabledIfEnvironmentVariable(named = "VERTEX_AI_GEMINI_LOCATION", matches = ".*") @@ -99,7 +98,7 @@ public class VertexAiGeminiPaymentTransactionIT { logger.info("" + content); } - @RepeatedTest(10) + @RepeatedTest(5) public void streamingPaymentStatuses() { Flux streamContent = this.chatClient.prompt() @@ -120,11 +119,9 @@ public class VertexAiGeminiPaymentTransactionIT { // Quota rate try { - Thread.sleep(20000); + Thread.sleep(2000); } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); } } @@ -189,7 +186,7 @@ public class VertexAiGeminiPaymentTransactionIT { return new VertexAiGeminiChatModel(vertexAi, VertexAiGeminiChatOptions.builder() - .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_PRO_1_5_FLASH) + .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_1_5_FLASH) // .withModel(VertexAiGeminiChatModel.ChatModel.GEMINI_PRO_1_5_PRO) .withTemperature(0.1f) // .withResponseMimeType(ResponseMimeType.JSON) diff --git a/pom.xml b/pom.xml index f9abde460..d021efb10 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 2.16.1 0.26.0 1.17.0 - 26.39.0 + 26.41.0 1.9.1 2.0.5 9.20.0 diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithFunctionBeanIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithFunctionBeanIT.java index e2375f340..8b0ad0ef1 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithFunctionBeanIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithFunctionBeanIT.java @@ -56,8 +56,8 @@ class FunctionCallWithFunctionBeanIT { contextRunner.withPropertyValues("spring.ai.vertex.ai.gemini.chat.options.model=" // + VertexAiGeminiChatModel.ChatModel.GEMINI_PRO.getValue()) - + VertexAiGeminiChatModel.ChatModel.GEMINI_PRO_1_5_PRO.getValue()) - // + VertexAiGeminiChatModel.ChatModel.GEMINI_PRO_1_5_FLASH.getValue()) + // + VertexAiGeminiChatModel.ChatModel.GEMINI_PRO_1_5_PRO.getValue()) + + VertexAiGeminiChatModel.ChatModel.GEMINI_1_5_FLASH.getValue()) .run(context -> { VertexAiGeminiChatModel chatModel = context.getBean(VertexAiGeminiChatModel.class); @@ -70,7 +70,7 @@ class FunctionCallWithFunctionBeanIT { var userMessage = new UserMessage( // "What's the weather like in San Francisco, Paris and in Tokyo? // Please let me know how many function calls you've preformed."); - "What's the weather like in San Francisco, Paris and in Tokyo?"); + "What's the weather like in San Francisco, Paris and in Tokyo? Perform multiple funciton execution if necessary. Return the temperature in Celsius."); ChatResponse response = chatModel.call(new Prompt(List.of(systemMessage, userMessage), VertexAiGeminiChatOptions.builder().withFunction("weatherFunction").build())); @@ -82,8 +82,6 @@ class FunctionCallWithFunctionBeanIT { assertThat(response.getResult().getOutput().getContent()).contains("30", "10", "15"); - Thread.sleep(10000); - response = chatModel.call(new Prompt(List.of(systemMessage, userMessage), VertexAiGeminiChatOptions.builder().withFunction("weatherFunction3").build())); diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithFunctionWrapperIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithFunctionWrapperIT.java index d8e3745ca..a9a2a724f 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithFunctionWrapperIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithFunctionWrapperIT.java @@ -55,7 +55,7 @@ public class FunctionCallWithFunctionWrapperIT { void functionCallTest() { contextRunner .withPropertyValues("spring.ai.vertex.ai.gemini.chat.options.model=" - + VertexAiGeminiChatModel.ChatModel.GEMINI_PRO_1_5_FLASH.getValue()) + + VertexAiGeminiChatModel.ChatModel.GEMINI_1_5_FLASH.getValue()) .run(context -> { VertexAiGeminiChatModel chatModel = context.getBean(VertexAiGeminiChatModel.class); diff --git a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithPromptFunctionIT.java b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithPromptFunctionIT.java index ec080bb96..a3c88082b 100644 --- a/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithPromptFunctionIT.java +++ b/spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vertexai/gemini/tool/FunctionCallWithPromptFunctionIT.java @@ -51,7 +51,7 @@ public class FunctionCallWithPromptFunctionIT { void functionCallTest() { contextRunner .withPropertyValues("spring.ai.vertex.ai.gemini.chat.options.model=" - + VertexAiGeminiChatModel.ChatModel.GEMINI_PRO_1_5_FLASH.getValue()) + + VertexAiGeminiChatModel.ChatModel.GEMINI_1_5_FLASH.getValue()) .run(context -> { VertexAiGeminiChatModel chatModel = context.getBean(VertexAiGeminiChatModel.class);