From f92a3f0fcb5710d0b3e611c88da5a40d83de02c5 Mon Sep 17 00:00:00 2001
From: Mark Pollack <mark.pollack@broadcom.com>
Date: Thu, 19 Dec 2024 15:14:18 -0500
Subject: [PATCH] Add support for general LLMs in FactCheckingEvaluator

The FactCheckingEvaluator was initially designed for the Bespoke Minicheck
model which doesn't require explicit instructions. This change adds support
for general-purpose LLMs like Claude and GPT-4 that need clear evaluation
instructions in their prompts.

Key changes:
- Introduce separate prompts for Bespoke and general LLMs
- Add factory method for Bespoke Minicheck configuration
- Make evaluation prompt configurable via constructor

This maintains backward compatibility while enabling the evaluator to work
effectively with any LLM implementation.
---
 .../ai/evaluation/FactCheckingEvaluator.java  | 33 +++++++++++++++++--
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/FactCheckingEvaluator.java b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/FactCheckingEvaluator.java
index 3dc03c769..6e8943e9f 100644
--- a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/FactCheckingEvaluator.java
+++ b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/FactCheckingEvaluator.java
@@ -64,19 +64,46 @@ import org.springframework.ai.chat.client.ChatClient;
 public class FactCheckingEvaluator implements Evaluator {
 
 	private static final String DEFAULT_EVALUATION_PROMPT_TEXT = """
+				Evaluate whether or not the following claim is supported by the provided document.
+				Respond with "yes" if the claim is supported, or "no" if it is not.
+				Document: \\n {document}\\n
+				Claim: \\n {claim}
+			""";
+
+	private static final String BESPOKE_EVALUATION_PROMPT_TEXT = """
 				Document: \\n {document}\\n
 				Claim: \\n {claim}
 			""";
 
 	private final ChatClient.Builder chatClientBuilder;
+	private final String evaluationPrompt;
 
 	/**
 	 * Constructs a new FactCheckingEvaluator with the provided ChatClient.Builder.
-	 * @param chatClientBuilder The builder for the ChatClient used to perform the
-	 * evaluation
+	 * Uses the default evaluation prompt suitable for general purpose LLMs.
+	 * @param chatClientBuilder The builder for the ChatClient used to perform the evaluation
 	 */
 	public FactCheckingEvaluator(ChatClient.Builder chatClientBuilder) {
+		this(chatClientBuilder, DEFAULT_EVALUATION_PROMPT_TEXT);
+	}
+
+	/**
+	 * Constructs a new FactCheckingEvaluator with the provided ChatClient.Builder and evaluation prompt.
+	 * @param chatClientBuilder The builder for the ChatClient used to perform the evaluation
+	 * @param evaluationPrompt The prompt text to use for evaluation
+	 */
+	public FactCheckingEvaluator(ChatClient.Builder chatClientBuilder, String evaluationPrompt) {
 		this.chatClientBuilder = chatClientBuilder;
+		this.evaluationPrompt = evaluationPrompt;
+	}
+
+	/**
+	 * Creates a FactCheckingEvaluator configured for use with the Bespoke Minicheck model.
+	 * @param chatClientBuilder The builder for the ChatClient used to perform the evaluation
+	 * @return A FactCheckingEvaluator configured for Bespoke Minicheck
+	 */
+	public static FactCheckingEvaluator forBespokeMinicheck(ChatClient.Builder chatClientBuilder) {
+		return new FactCheckingEvaluator(chatClientBuilder, BESPOKE_EVALUATION_PROMPT_TEXT);
 	}
 
 	/**
@@ -94,7 +121,7 @@ public class FactCheckingEvaluator implements Evaluator {
 
 		String evaluationResponse = this.chatClientBuilder.build()
 			.prompt()
-			.user(userSpec -> userSpec.text(DEFAULT_EVALUATION_PROMPT_TEXT)
+			.user(userSpec -> userSpec.text(evaluationPrompt)
 				.param("document", context)
 				.param("claim", response))
 			.call()