diff --git a/pom.xml b/pom.xml index c2a7ef0ce..59229491d 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ spring-ai-azure-openai spring-ai-ollama spring-ai-huggingface + spring-ai-test spring-ai-spring-boot-autoconfigure spring-ai-spring-boot-starters/spring-ai-starter-openai spring-ai-spring-boot-starters/spring-ai-starter-azure-openai @@ -71,6 +72,8 @@ UTF-8 UTF-8 17 + 17 + 17 3.1.3 diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIT.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIT.java index a6e854131..cd7dc7d53 100644 --- a/spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIT.java +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIT.java @@ -56,7 +56,7 @@ class ClientIT extends AbstractIT { PromptTemplate promptTemplate = new PromptTemplate(template, Map.of("subject", "ice cream flavors", "format", format)); Prompt prompt = new Prompt(promptTemplate.createMessage()); - Generation generation = openAiClient.generate(prompt).getGeneration(); + Generation generation = this.openAiClient.generate(prompt).getGeneration(); List list = outputParser.parse(generation.getText()); System.out.println(list); diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java index 1c06eb36e..8cec9f7e0 100644 --- a/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java @@ -2,8 +2,8 @@ package org.springframework.ai.openai.testutils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.ai.client.AiClient; import org.springframework.ai.client.AiResponse; -import org.springframework.ai.openai.client.OpenAiClient; import org.springframework.ai.prompt.Prompt; import org.springframework.ai.prompt.PromptTemplate; import org.springframework.ai.prompt.messages.Message; @@ -23,7 +23,7 @@ public abstract class AbstractIT { private static final Logger logger = LoggerFactory.getLogger(AbstractIT.class); @Autowired - protected OpenAiClient openAiClient; + protected AiClient openAiClient; @Value("classpath:/prompts/eval/qa-evaluator-accurate-answer.st") protected Resource qaEvaluatorAccurateAnswerResource; @@ -67,4 +67,4 @@ public abstract class AbstractIT { } } -} +} \ No newline at end of file diff --git a/spring-ai-test/README.md b/spring-ai-test/README.md new file mode 100644 index 000000000..cacd5c365 --- /dev/null +++ b/spring-ai-test/README.md @@ -0,0 +1,2 @@ +TODO: + Documentation and sample tests using the `BasicEvaluationTest``. \ No newline at end of file diff --git a/spring-ai-test/pom.xml b/spring-ai-test/pom.xml new file mode 100644 index 000000000..1da1031e2 --- /dev/null +++ b/spring-ai-test/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + org.springframework.experimental.ai + spring-ai + 0.7.0-SNAPSHOT + + spring-ai-test + jar + Spring AI Test + Test support for AI programming + https://github.com/spring-projects-experimental/spring-ai + + + https://github.com/spring-projects-experimental/spring-ai + git://github.com/spring-projects-experimental/spring-ai.git + git@github.com:spring-projects-experimental/spring-ai.git + + + + 17 + 17 + + + + + org.springframework.experimental.ai + spring-ai-openai + ${project.parent.version} + true + + + + org.springframework.boot + spring-boot-starter-test + + + \ No newline at end of file diff --git a/spring-ai-test/src/main/java/org/springframework/ai/evaluation/BasicEvaluationTest.java b/spring-ai-test/src/main/java/org/springframework/ai/evaluation/BasicEvaluationTest.java new file mode 100644 index 000000000..25aeaec35 --- /dev/null +++ b/spring-ai-test/src/main/java/org/springframework/ai/evaluation/BasicEvaluationTest.java @@ -0,0 +1,86 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.evaluation; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.client.AiClient; +import org.springframework.ai.client.AiResponse; +import org.springframework.ai.prompt.Prompt; +import org.springframework.ai.prompt.PromptTemplate; +import org.springframework.ai.prompt.messages.Message; +import org.springframework.ai.prompt.messages.SystemMessage; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class BasicEvaluationTest { + + private static final Logger logger = LoggerFactory.getLogger(BasicEvaluationTest.class); + + @Autowired + protected AiClient openAiClient; + + @Value("classpath:/prompts/spring/test/evaluation/qa-evaluator-accurate-answer.st") + protected Resource qaEvaluatorAccurateAnswerResource; + + @Value("classpath:/prompts/spring/test/evaluation/qa-evaluator-not-related-message.st") + protected Resource qaEvaluatorNotRelatedResource; + + @Value("classpath:/prompts/spring/test/evaluation/qa-evaluator-fact-based-answer.st") + protected Resource qaEvaluatorFactBasedAnswerResource; + + @Value("classpath:/prompts/spring/test/evaluation/user-evaluator-message.st") + protected Resource userEvaluatorResource; + + protected void evaluateQuestionAndAnswer(String question, AiResponse response, boolean factBased) { + assertThat(response).isNotNull(); + String answer = response.getGeneration().getText(); + logger.info("Question: " + question); + logger.info("Answer:" + answer); + PromptTemplate userPromptTemplate = new PromptTemplate(userEvaluatorResource, + Map.of("question", question, "answer", answer)); + SystemMessage systemMessage; + if (factBased) { + systemMessage = new SystemMessage(qaEvaluatorFactBasedAnswerResource); + } + else { + systemMessage = new SystemMessage(qaEvaluatorAccurateAnswerResource); + } + Message userMessage = userPromptTemplate.createMessage(); + Prompt prompt = new Prompt(List.of(userMessage, systemMessage)); + String yesOrNo = openAiClient.generate(prompt).getGeneration().getText(); + logger.info("Is Answer related to question: " + yesOrNo); + if (yesOrNo.equalsIgnoreCase("no")) { + SystemMessage notRelatedSystemMessage = new SystemMessage(qaEvaluatorNotRelatedResource); + prompt = new Prompt(List.of(userMessage, notRelatedSystemMessage)); + String reasonForFailure = openAiClient.generate(prompt).getGeneration().getText(); + fail(reasonForFailure); + } + else { + logger.info("Answer is related to question."); + assertThat(yesOrNo).isEqualTo("YES"); + } + } + +} \ No newline at end of file diff --git a/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-accurate-answer.st b/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-accurate-answer.st new file mode 100644 index 000000000..562703595 --- /dev/null +++ b/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-accurate-answer.st @@ -0,0 +1,3 @@ +You are an AI assistant who helps users to evaluate if the answers to questions are accurate. +You will be provided with a QUESTION and an ANSWER. +Your goal is to evaluate the QUESTION and ANSWER and reply with a YES or NO answer. \ No newline at end of file diff --git a/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-fact-based-answer.st b/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-fact-based-answer.st new file mode 100644 index 000000000..22fc3e88d --- /dev/null +++ b/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-fact-based-answer.st @@ -0,0 +1,7 @@ +You are an AI evaluator. Your task is to verify if the provided ANSWER is a direct and accurate response to the given QUESTION. If the ANSWER is correct and directly answers the QUESTION, reply with "YES". If the ANSWER is not a direct response or is inaccurate, reply with "NO". + +For example: + +If the QUESTION is "What is the capital of France?" and the ANSWER is "Paris.", you should respond with "YES". +If the QUESTION is "What is the capital of France?" and the ANSWER is "France is in Europe.", respond with "NO". +Now, evaluate the following: diff --git a/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-not-related-message.st b/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-not-related-message.st new file mode 100644 index 000000000..7c33e675e --- /dev/null +++ b/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-not-related-message.st @@ -0,0 +1,4 @@ +You are an AI assistant who helps users to evaluate if the answers to questions are accurate. +You will be provided with a QUESTION and an ANSWER. +A previous evaluation has determined that QUESTION and ANSWER are not related. +Give an explanation as to why they are not related. \ No newline at end of file diff --git a/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/user-evaluator-message.st b/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/user-evaluator-message.st new file mode 100644 index 000000000..b3fa3e902 --- /dev/null +++ b/spring-ai-test/src/main/resources/prompts/spring/test/evaluation/user-evaluator-message.st @@ -0,0 +1,6 @@ +The question and answer to evaluate are: + +QUESTION: ```{question}``` + +ANSWER: ```{answer}``` +