Add test support for AI programming

pom.xml

@@ -18,6 +18,7 @@
 		<module>spring-ai-azure-openai</module>
 		<module>spring-ai-ollama</module>
 		<module>spring-ai-huggingface</module>
+		<module>spring-ai-test</module>
 		<module>spring-ai-spring-boot-autoconfigure</module>
 		<module>spring-ai-spring-boot-starters/spring-ai-starter-openai</module>
 		<module>spring-ai-spring-boot-starters/spring-ai-starter-azure-openai</module>
@@ -71,6 +72,8 @@
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 		<java.version>17</java.version>
+		<maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
 
 		<!-- prodution dependencies -->
 		<spring-boot.version>3.1.3</spring-boot.version>

spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIT.java

@@ -56,7 +56,7 @@ class ClientIT extends AbstractIT {
 		PromptTemplate promptTemplate = new PromptTemplate(template,
 				Map.of("subject", "ice cream flavors", "format", format));
 		Prompt prompt = new Prompt(promptTemplate.createMessage());
-		Generation generation = openAiClient.generate(prompt).getGeneration();
+		Generation generation = this.openAiClient.generate(prompt).getGeneration();
 
 		List<String> list = outputParser.parse(generation.getText());
 		System.out.println(list);

spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIT.java

@@ -2,8 +2,8 @@ package org.springframework.ai.openai.testutils;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.ai.client.AiClient;
 import org.springframework.ai.client.AiResponse;
-import org.springframework.ai.openai.client.OpenAiClient;
 import org.springframework.ai.prompt.Prompt;
 import org.springframework.ai.prompt.PromptTemplate;
 import org.springframework.ai.prompt.messages.Message;
@@ -23,7 +23,7 @@ public abstract class AbstractIT {
 	private static final Logger logger = LoggerFactory.getLogger(AbstractIT.class);
 
 	@Autowired
-	protected OpenAiClient openAiClient;
+	protected AiClient openAiClient;
 
 	@Value("classpath:/prompts/eval/qa-evaluator-accurate-answer.st")
 	protected Resource qaEvaluatorAccurateAnswerResource;
@@ -67,4 +67,4 @@ public abstract class AbstractIT {
 		}
 	}
 
-}
+}

spring-ai-test/README.md

@@ -0,0 +1,2 @@
+TODO:
+	Documentation and sample tests using the `BasicEvaluationTest``.

spring-ai-test/pom.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.springframework.experimental.ai</groupId>
+		<artifactId>spring-ai</artifactId>
+		<version>0.7.0-SNAPSHOT</version>
+	</parent>
+	<artifactId>spring-ai-test</artifactId>
+	<packaging>jar</packaging>
+	<name>Spring AI Test</name>
+	<description>Test support for AI programming</description>
+	<url>https://github.com/spring-projects-experimental/spring-ai</url>
+
+	<scm>
+		<url>https://github.com/spring-projects-experimental/spring-ai</url>
+		<connection>git://github.com/spring-projects-experimental/spring-ai.git</connection>
+		<developerConnection>git@github.com:spring-projects-experimental/spring-ai.git</developerConnection>
+	</scm>
+
+	<properties>
+		<maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+	</properties>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.springframework.experimental.ai</groupId>
+			<artifactId>spring-ai-openai</artifactId>
+			<version>${project.parent.version}</version>
+			<optional>true</optional>
+		</dependency>
+
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-test</artifactId>
+		</dependency>
+	</dependencies>
+</project>

spring-ai-test/src/main/java/org/springframework/ai/evaluation/BasicEvaluationTest.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.evaluation;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.client.AiClient;
+import org.springframework.ai.client.AiResponse;
+import org.springframework.ai.prompt.Prompt;
+import org.springframework.ai.prompt.PromptTemplate;
+import org.springframework.ai.prompt.messages.Message;
+import org.springframework.ai.prompt.messages.SystemMessage;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.Resource;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+public class BasicEvaluationTest {
+
+	private static final Logger logger = LoggerFactory.getLogger(BasicEvaluationTest.class);
+
+	@Autowired
+	protected AiClient openAiClient;
+
+	@Value("classpath:/prompts/spring/test/evaluation/qa-evaluator-accurate-answer.st")
+	protected Resource qaEvaluatorAccurateAnswerResource;
+
+	@Value("classpath:/prompts/spring/test/evaluation/qa-evaluator-not-related-message.st")
+	protected Resource qaEvaluatorNotRelatedResource;
+
+	@Value("classpath:/prompts/spring/test/evaluation/qa-evaluator-fact-based-answer.st")
+	protected Resource qaEvaluatorFactBasedAnswerResource;
+
+	@Value("classpath:/prompts/spring/test/evaluation/user-evaluator-message.st")
+	protected Resource userEvaluatorResource;
+
+	protected void evaluateQuestionAndAnswer(String question, AiResponse response, boolean factBased) {
+		assertThat(response).isNotNull();
+		String answer = response.getGeneration().getText();
+		logger.info("Question: " + question);
+		logger.info("Answer:" + answer);
+		PromptTemplate userPromptTemplate = new PromptTemplate(userEvaluatorResource,
+				Map.of("question", question, "answer", answer));
+		SystemMessage systemMessage;
+		if (factBased) {
+			systemMessage = new SystemMessage(qaEvaluatorFactBasedAnswerResource);
+		}
+		else {
+			systemMessage = new SystemMessage(qaEvaluatorAccurateAnswerResource);
+		}
+		Message userMessage = userPromptTemplate.createMessage();
+		Prompt prompt = new Prompt(List.of(userMessage, systemMessage));
+		String yesOrNo = openAiClient.generate(prompt).getGeneration().getText();
+		logger.info("Is Answer related to question: " + yesOrNo);
+		if (yesOrNo.equalsIgnoreCase("no")) {
+			SystemMessage notRelatedSystemMessage = new SystemMessage(qaEvaluatorNotRelatedResource);
+			prompt = new Prompt(List.of(userMessage, notRelatedSystemMessage));
+			String reasonForFailure = openAiClient.generate(prompt).getGeneration().getText();
+			fail(reasonForFailure);
+		}
+		else {
+			logger.info("Answer is related to question.");
+			assertThat(yesOrNo).isEqualTo("YES");
+		}
+	}
+
+}

spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-accurate-answer.st

@@ -0,0 +1,3 @@
+You are an AI assistant who helps users to evaluate if the answers to questions are accurate.
+You will be provided with a QUESTION and an ANSWER.
+Your goal is to evaluate the QUESTION and ANSWER and reply with a YES or NO answer.

spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-fact-based-answer.st

@@ -0,0 +1,7 @@
+You are an AI evaluator. Your task is to verify if the provided ANSWER is a direct and accurate response to the given QUESTION. If the ANSWER is correct and directly answers the QUESTION, reply with "YES". If the ANSWER is not a direct response or is inaccurate, reply with "NO".
+
+For example:
+
+If the QUESTION is "What is the capital of France?" and the ANSWER is "Paris.", you should respond with "YES".
+If the QUESTION is "What is the capital of France?" and the ANSWER is "France is in Europe.", respond with "NO".
+Now, evaluate the following:

spring-ai-test/src/main/resources/prompts/spring/test/evaluation/qa-evaluator-not-related-message.st

@@ -0,0 +1,4 @@
+You are an AI assistant who helps users to evaluate if the answers to questions are accurate.
+You will be provided with a QUESTION and an ANSWER.
+A previous evaluation has determined that QUESTION and ANSWER are not related.
+Give an explanation as to why they are not related.

spring-ai-test/src/main/resources/prompts/spring/test/evaluation/user-evaluator-message.st

@@ -0,0 +1,6 @@
+The question and answer to evaluate are:
+
+QUESTION: ```{question}```
+
+ANSWER: ```{answer}```
+