Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve acme bike use case and evaluation tests

Browse Source
This commit is contained in:
Mark Pollack
2023-08-19 12:31:48 -04:00
parent 2b10843537
commit 8e276cae6d
15 changed files with 191 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
spring-ai-core/src/main/java/org/springframework/ai/document/Document.java
View File

@@ -39,7 +39,7 @@ public class Document {
private String metadataTemplate = DEFAULT_METADATA_TEMPLATE;
private String metadataSeperator;
private String metadataSeperator = "\n";
public Document(String text) {
this.text = text;
@@ -132,4 +132,16 @@ public class Document {
return "Document{" + "id='" + id + '\'' + ", metadata=" + metadata + ", text='" + text + '\'' + '}';
}
public void setTextTemplate(String textTemplate) {
this.textTemplate = textTemplate;
}
public void setMetadataTemplate(String metadataTemplate) {
this.metadataTemplate = metadataTemplate;
}
public void setMetadataSeperator(String metadataSeperator) {
this.metadataSeperator = metadataSeperator;
}
}

28
spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonLoader.java
View File

@@ -16,19 +16,19 @@ public class JsonLoader implements Loader {
/**
* The key from the JSON that we will use as the text to parse into the Document text
*/
private String textKey = "text";
private List<String> jsonKeysToUse = new ArrayList<>();
private Resource resource;
public JsonLoader(Resource resource) {
Objects.requireNonNull(this.resource, "The Spring Resource must not be null");
Objects.requireNonNull(resource, "The Spring Resource must not be null");
this.resource = resource;
}
public JsonLoader(String textKey, Resource resource) {
Objects.requireNonNull(textKey, "textKey must not be null");
public JsonLoader(Resource resource, String... jsonKeysToUse) {
Objects.requireNonNull(jsonKeysToUse, "keys must not be null");
Objects.requireNonNull(resource, "The Spring Resource must not be null");
this.textKey = textKey;
this.jsonKeysToUse = List.of(jsonKeysToUse);
this.resource = resource;
}
@@ -51,8 +51,22 @@ public class JsonLoader implements Loader {
new TypeReference<List<Map<String, Object>>>() {
});
for (Map<String, Object> item : jsonData) {
if (item.containsKey(this.textKey)) {
Document document = new Document(item.get(this.textKey).toString());
StringBuilder sb = new StringBuilder();
for (String key : jsonKeysToUse) {
if (item.containsKey(key)) {
sb.append(key);
sb.append(": ");
sb.append(item.get(key));
sb.append(System.lineSeparator());
}
}
if (!sb.isEmpty()) {
Document document = new Document(sb.toString());
documents.add(document);
}
else {
Document document = new Document(item.toString());
documents.add(document);
}
}

37
spring-ai-core/src/main/java/org/springframework/ai/splitter/TokenTextSplitter.java
View File

@@ -13,17 +13,25 @@ import java.util.List;
*/
public class TokenTextSplitter extends TextSplitter {
private static final int DEFAULT_CHUNK_SIZE = 800; // The target size of each text
// chunk in tokens
private final int defaultChunkSize = 800; // The target size of each text
// chunk in tokens
private static final int MIN_CHUNK_SIZE_CHARS = 350; // The minimum size of each text
// chunk in characters
private int minChunkSizeChars = 350; // The minimum size of each text
// chunk in characters
private static final int MIN_CHUNK_LENGTH_TO_EMBED = 5; // Discard chunks shorter than
// this
private int minChunkLengthToEmbed = 5; // Discard chunks shorter than this
private static final int MAX_NUM_CHUNKS = 10000; // The maximum number of chunks to
// generate from a text
private int maxNumChunks = 10000; // The maximum number of chunks to generate from a
// text
private boolean keepSeparator = true;
public TokenTextSplitter() {
}
public TokenTextSplitter(boolean keepSeparator) {
this.keepSeparator = keepSeparator;
}
private final EncodingRegistry registry = Encodings.newLazyEncodingRegistry();
@@ -31,7 +39,7 @@ public class TokenTextSplitter extends TextSplitter {
@Override
protected List<String> splitText(String text) {
return split(text, DEFAULT_CHUNK_SIZE);
return split(text, defaultChunkSize);
}
public List<String> split(String text, int chunkSize) {
@@ -42,7 +50,7 @@ public class TokenTextSplitter extends TextSplitter {
List<Integer> tokens = getEncodedTokens(text);
List<String> chunks = new ArrayList<>();
int num_chunks = 0;
while (!tokens.isEmpty() && num_chunks < MAX_NUM_CHUNKS) {
while (!tokens.isEmpty() && num_chunks < maxNumChunks) {
List<Integer> chunk = tokens.subList(0, Math.min(chunkSize, tokens.size()));
String chunkText = decodeTokens(chunk);
@@ -56,12 +64,13 @@ public class TokenTextSplitter extends TextSplitter {
int lastPunctuation = Math.max(chunkText.lastIndexOf('.'), Math.max(chunkText.lastIndexOf('?'),
Math.max(chunkText.lastIndexOf('!'), chunkText.lastIndexOf('\n'))));
if (lastPunctuation != -1 && lastPunctuation > MIN_CHUNK_SIZE_CHARS) {
if (lastPunctuation != -1 && lastPunctuation > minChunkSizeChars) {
// Truncate the chunk text at the punctuation mark
chunkText = chunkText.substring(0, lastPunctuation + 1);
}
String chunk_text_to_append = chunkText.replace("\n", " ").trim();
if (chunk_text_to_append.length() > MIN_CHUNK_LENGTH_TO_EMBED) {
String chunk_text_to_append = (this.keepSeparator) ? chunkText.trim() : chunkText.replace("\n", " ").trim();
if (chunk_text_to_append.length() > minChunkLengthToEmbed) {
chunks.add(chunk_text_to_append);
}
@@ -74,7 +83,7 @@ public class TokenTextSplitter extends TextSplitter {
// Handle the remaining tokens
if (!tokens.isEmpty()) {
String remaining_text = decodeTokens(tokens).replace("\n", " ").trim();
if (remaining_text.length() > MIN_CHUNK_LENGTH_TO_EMBED) {
if (remaining_text.length() > minChunkLengthToEmbed) {
chunks.add(remaining_text);
}
}

2
spring-ai-core/src/test/java/org/springframework/ai/loader/LoaderTests.java
View File

@@ -20,7 +20,7 @@ public class LoaderTests {
@Test
void loadJson() {
assertThat(resource).isNotNull();
JsonLoader jsonLoader = new JsonLoader("description", resource);
JsonLoader jsonLoader = new JsonLoader(resource, "description");
List<Document> documents = jsonLoader.load();
assertThat(documents).isNotEmpty();
for (Document document : documents) {

2
spring-ai-openai/src/main/java/org/springframework/ai/openai/client/OpenAiClient.java
View File

@@ -41,7 +41,7 @@ public class OpenAiClient implements AiClient {
// TODO how to set default options for the entire client
// TODO expose request options into Prompt API via PromptOptions
private Double temperature = 0.5;
private Double temperature = 0.7;
private String model = "gpt-3.5-turbo";

1
spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java
View File

@@ -27,6 +27,7 @@ public class OpenAiTestConfiguration {
@Bean
public OpenAiClient openAiClient(OpenAiService theoOpenAiService) {
OpenAiClient openAiClient = new OpenAiClient(theoOpenAiService);
openAiClient.setTemperature(0.3);
return openAiClient;
}

56
spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIntegrationTest.java
View File

@@ -1,32 +1,42 @@
package org.springframework.ai.openai.acme;
import org.junit.jupiter.api.Test;
import org.springframework.ai.document.Document;
import org.springframework.ai.client.AiResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.client.AiClient;
import org.springframework.ai.client.AiResponse;
import org.springframework.ai.document.Document;
import org.springframework.ai.loader.impl.JsonLoader;
import org.springframework.ai.openai.embedding.OpenAiEmbeddingClient;
import org.springframework.ai.openai.testutils.AbstractIntegrationTest;
import org.springframework.ai.prompt.Prompt;
import org.springframework.ai.prompt.messages.SystemMessage;
import org.springframework.ai.prompt.SystemPromptTemplate;
import org.springframework.ai.prompt.messages.Message;
import org.springframework.ai.prompt.messages.UserMessage;
import org.springframework.ai.retriever.impl.VectorStoreRetriever;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.impl.InMemoryVectorStore;
import org.springframework.ai.openai.embedding.OpenAiEmbeddingClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.io.Resource;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
@SpringBootTest
public class AcmeIntegrationTest {
public class AcmeIntegrationTest extends AbstractIntegrationTest {
@Value("classpath:bikes.json")
private Resource resource;
private static final Logger logger = LoggerFactory.getLogger(AcmeIntegrationTest.class);
@Value("classpath:/data/acme/bikes.json")
private Resource bikesResource;
@Value("classpath:/prompts/acme/system-qa.st")
private Resource systemBikePrompt;
@Autowired
private OpenAiEmbeddingClient embeddingClient;
@@ -36,19 +46,21 @@ public class AcmeIntegrationTest {
@Test
void beanTest() {
assertThat(resource).isNotNull();
assertThat(bikesResource).isNotNull();
assertThat(embeddingClient).isNotNull();
assertThat(aiClient).isNotNull();
}
// @Test
void acmeChain() {
// Step 1 - load documents
JsonLoader jsonLoader = new JsonLoader("description", resource);
JsonLoader jsonLoader = new JsonLoader(bikesResource, "name", "price", "shortDescription", "description");
List<Document> documents = jsonLoader.load();
// Step 2 - Create embeddings and save to vector store
logger.info("Creating Embeddings...");
VectorStore vectorStore = new InMemoryVectorStore(embeddingClient);
vectorStore.add(documents);
@@ -56,38 +68,46 @@ public class AcmeIntegrationTest {
// Now user query
// This will be wrapped up in a chain
VectorStoreRetriever vectorStoreRetriever = new VectorStoreRetriever(vectorStore);
String userQuery = "What bike is good for city commuting?";
logger.info("Retrieving relevant documents");
String userQuery = "How much does the SonicRide 8S cost?";
// "Tell me about the bike 'The SonicRide 8S'" ;
// "What bike is good for city commuting?";
List<Document> similarDocuments = vectorStoreRetriever.retrieve(userQuery);
logger.info(String.format("Found %s relevant documents.", similarDocuments.size()));
// Try the case where not product was specified, so query over whatever docs might
// be releveant.
SystemMessage systemMessage = getSystemMessage(similarDocuments);
Message systemMessage = getSystemMessage(similarDocuments);
UserMessage userMessage = new UserMessage(userQuery);
// Create the prompt ad-hoc for now, need to put in system message and user
// message via ChatPromptTemplate or some other message building mechanic
// message via ChatPromptTemplate or some other message building mechanic;
logger.info("Asking AI model to reply to question.");
Prompt prompt = new Prompt(List.of(systemMessage, userMessage));
logger.info("AI responded.");
AiResponse response = aiClient.generate(prompt);
evaluateQuestionAndAnswer(userQuery, response, true);
// Chain
// qa = new ConversationalRetrievalChain(llmClient, userPromptTemplate,
// vectorStoreRetriever, )
}
private SystemMessage getSystemMessage(List<Document> similarDocuments) {
private Message getSystemMessage(List<Document> similarDocuments) {
// Would need to figure out which of the documenta metadata fields to add, from
// the loader, now just the 'full description.'
String systemMessageText = similarDocuments.stream()
.map(entry -> entry.getContent())
.collect(Collectors.joining("\n"));
String documents = similarDocuments.stream().map(entry -> entry.getContent()).collect(Collectors.joining("\n"));
return new SystemMessage(systemMessageText);
SystemPromptTemplate systemPromptTemplate = new SystemPromptTemplate(systemBikePrompt);
Message systemMessage = systemPromptTemplate.createMessage(Map.of("documents", documents));
return systemMessage;
}

33
spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIntegrationTests.java
View File

@@ -1,16 +1,16 @@
package org.springframework.ai.openai.client;
import org.junit.jupiter.api.Test;
import org.springframework.ai.client.AiResponse;
import org.springframework.ai.client.Generation;
import org.springframework.ai.openai.testutils.AbstractIntegrationTest;
import org.springframework.ai.parser.JsonOutputParser;
import org.springframework.ai.parser.ListOutputParser;
import org.springframework.ai.prompt.Prompt;
import org.springframework.ai.prompt.PromptTemplate;
import org.springframework.ai.prompt.SystemPromptTemplate;
import org.springframework.ai.prompt.messages.Message;
import org.springframework.ai.prompt.messages.SystemMessage;
import org.springframework.ai.prompt.messages.UserMessage;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.convert.support.DefaultConversionService;
@@ -24,20 +24,11 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@SpringBootTest
class ClientIntegrationTests {
@Autowired
OpenAiClient openAiClient;
class ClientIntegrationTests extends AbstractIntegrationTest {
@Value("classpath:/prompts/system-message.st")
private Resource systemResource;
@Value("classpath:/prompts/system-evaluator-message.st")
private Resource systemEvaluatorResource;
@Value("classpath:/prompts/user-evaluator-message.st")
private Resource userEvaluatorResource;
@Test
void roleTest() {
String request = "Tell me about 3 famous pirates from the Golden Age of Piracy and why they did.";
@@ -47,22 +38,8 @@ class ClientIntegrationTests {
SystemPromptTemplate systemPromptTemplate = new SystemPromptTemplate(systemResource);
Message systemMessage = systemPromptTemplate.createMessage(Map.of("name", name, "voice", voice));
Prompt prompt = new Prompt(List.of(userMessage, systemMessage));
Generation response = openAiClient.generate(prompt).getGeneration();
System.out.println(response);
assertThat(response).isNotNull();
evaluateQuestionAndAnswer(request, response.getText());
}
private void evaluateQuestionAndAnswer(String question, String answer) {
PromptTemplate userPromptTemplate = new PromptTemplate(userEvaluatorResource,
Map.of("question", question, "answer", answer));
SystemMessage systemMessage = new SystemMessage(systemEvaluatorResource);
Message userMessage = userPromptTemplate.createMessage();
Prompt prompt = new Prompt(List.of(userMessage, systemMessage));
Generation response = openAiClient.generate(prompt).getGeneration();
System.out.println(response);
assertThat(response.getText()).isEqualTo("YES");
AiResponse response = openAiClient.generate(prompt);
evaluateQuestionAndAnswer(request, response, false);
}
@Test

70
spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIntegrationTest.java Normal file
View File

@@ -0,0 +1,70 @@
package org.springframework.ai.openai.testutils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.client.AiResponse;
import org.springframework.ai.openai.client.OpenAiClient;
import org.springframework.ai.prompt.Prompt;
import org.springframework.ai.prompt.PromptTemplate;
import org.springframework.ai.prompt.messages.Message;
import org.springframework.ai.prompt.messages.SystemMessage;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
public class AbstractIntegrationTest {
private static final Logger logger = LoggerFactory.getLogger(AbstractIntegrationTest.class);
@Autowired
protected OpenAiClient openAiClient;
@Value("classpath:/prompts/eval/qa-evaluator-accurate-answer.st")
protected Resource qaEvaluatorAccurateAnswerResource;
@Value("classpath:/prompts/eval/qa-evaluator-not-related-message.st")
protected Resource qaEvaluatorNotRelatedResource;
@Value("classpath:/prompts/eval/qa-evaluator-fact-based-answer.st")
protected Resource qaEvalutaorFactBasedAnswerResource;
@Value("classpath:/prompts/eval/user-evaluator-message.st")
protected Resource userEvaluatorResource;
protected void evaluateQuestionAndAnswer(String question, AiResponse response, boolean factBased) {
assertThat(response).isNotNull();
String answer = response.getGeneration().getText();
logger.info("Question: " + question);
logger.info("Answer:" + answer);
PromptTemplate userPromptTemplate = new PromptTemplate(userEvaluatorResource,
Map.of("question", question, "answer", answer));
SystemMessage systemMessage;
if (factBased) {
systemMessage = new SystemMessage(qaEvalutaorFactBasedAnswerResource);
}
else {
systemMessage = new SystemMessage(qaEvaluatorAccurateAnswerResource);
}
Message userMessage = userPromptTemplate.createMessage();
Prompt prompt = new Prompt(List.of(userMessage, systemMessage));
String yesOrNo = openAiClient.generate(prompt).getGeneration().getText();
logger.info("Is Answer related to question: " + yesOrNo);
if (yesOrNo.equalsIgnoreCase("no")) {
SystemMessage notRelatedSystemMessage = new SystemMessage(qaEvaluatorNotRelatedResource);
prompt = new Prompt(List.of(userMessage, notRelatedSystemMessage));
String reasonForFailure = openAiClient.generate(prompt).getGeneration().getText();
fail(reasonForFailure);
}
else {
logger.info("Answer is related to question.");
assertThat(yesOrNo).isEqualTo("YES");
}
}
}

0
spring-ai-openai/src/test/resources/bikes.json → spring-ai-openai/src/test/resources/data/acme/bikes.json
View File

7
spring-ai-openai/src/test/resources/prompts/acme/system-qa.st Normal file
View File

@@ -0,0 +1,7 @@
You're assisting with questions about products in a bicycle catalog.
Use the information from the DOCUMENTS section to provide accurate answers.
The the answer involves referring to the price or the dimension of the bicycle, include the bicycle name in the response.
If unsure, simply state that you don't know.
DOCUMENTS:
{documents}

0
spring-ai-openai/src/test/resources/prompts/system-evaluator-message.st → spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-accurate-answer.st
View File

7
spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-fact-based-answer.st Normal file
View File

@@ -0,0 +1,7 @@
You are an AI evaluator. Your task is to verify if the provided ANSWER is a direct and accurate response to the given QUESTION. If the ANSWER is correct and directly answers the QUESTION, reply with "YES". If the ANSWER is not a direct response or is inaccurate, reply with "NO".
For example:
If the QUESTION is "What is the capital of France?" and the ANSWER is "Paris.", you should respond with "YES".
If the QUESTION is "What is the capital of France?" and the ANSWER is "France is in Europe.", respond with "NO".
Now, evaluate the following:

4
spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-not-related-message.st Normal file
View File

@@ -0,0 +1,4 @@
You are an AI assistant who helps users to evaluate if the answers to questions are accurate.
You will be provided with a QUESTION and an ANSWER.
A previous evaluation has determined that QUESTION and ANSWER are not related.
Give an explanation as to why they are not related.

0
spring-ai-openai/src/test/resources/prompts/user-evaluator-message.st → spring-ai-openai/src/test/resources/prompts/eval/user-evaluator-message.st
View File