diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/Document.java b/spring-ai-core/src/main/java/org/springframework/ai/document/Document.java index fd0b600c4..b5796753e 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/document/Document.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/document/Document.java @@ -39,7 +39,7 @@ public class Document { private String metadataTemplate = DEFAULT_METADATA_TEMPLATE; - private String metadataSeperator; + private String metadataSeperator = "\n"; public Document(String text) { this.text = text; @@ -132,4 +132,16 @@ public class Document { return "Document{" + "id='" + id + '\'' + ", metadata=" + metadata + ", text='" + text + '\'' + '}'; } + public void setTextTemplate(String textTemplate) { + this.textTemplate = textTemplate; + } + + public void setMetadataTemplate(String metadataTemplate) { + this.metadataTemplate = metadataTemplate; + } + + public void setMetadataSeperator(String metadataSeperator) { + this.metadataSeperator = metadataSeperator; + } + } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonLoader.java b/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonLoader.java index d3609742d..ec6152dfd 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonLoader.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonLoader.java @@ -16,19 +16,19 @@ public class JsonLoader implements Loader { /** * The key from the JSON that we will use as the text to parse into the Document text */ - private String textKey = "text"; + private List jsonKeysToUse = new ArrayList<>(); private Resource resource; public JsonLoader(Resource resource) { - Objects.requireNonNull(this.resource, "The Spring Resource must not be null"); + Objects.requireNonNull(resource, "The Spring Resource must not be null"); this.resource = resource; } - public JsonLoader(String textKey, Resource resource) { - Objects.requireNonNull(textKey, "textKey must not be null"); + public JsonLoader(Resource resource, String... jsonKeysToUse) { + Objects.requireNonNull(jsonKeysToUse, "keys must not be null"); Objects.requireNonNull(resource, "The Spring Resource must not be null"); - this.textKey = textKey; + this.jsonKeysToUse = List.of(jsonKeysToUse); this.resource = resource; } @@ -51,8 +51,22 @@ public class JsonLoader implements Loader { new TypeReference>>() { }); for (Map item : jsonData) { - if (item.containsKey(this.textKey)) { - Document document = new Document(item.get(this.textKey).toString()); + StringBuilder sb = new StringBuilder(); + for (String key : jsonKeysToUse) { + if (item.containsKey(key)) { + sb.append(key); + sb.append(": "); + sb.append(item.get(key)); + sb.append(System.lineSeparator()); + } + } + + if (!sb.isEmpty()) { + Document document = new Document(sb.toString()); + documents.add(document); + } + else { + Document document = new Document(item.toString()); documents.add(document); } } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/splitter/TokenTextSplitter.java b/spring-ai-core/src/main/java/org/springframework/ai/splitter/TokenTextSplitter.java index 4614e3af2..d2d92cbbc 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/splitter/TokenTextSplitter.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/splitter/TokenTextSplitter.java @@ -13,17 +13,25 @@ import java.util.List; */ public class TokenTextSplitter extends TextSplitter { - private static final int DEFAULT_CHUNK_SIZE = 800; // The target size of each text - // chunk in tokens + private final int defaultChunkSize = 800; // The target size of each text + // chunk in tokens - private static final int MIN_CHUNK_SIZE_CHARS = 350; // The minimum size of each text - // chunk in characters + private int minChunkSizeChars = 350; // The minimum size of each text + // chunk in characters - private static final int MIN_CHUNK_LENGTH_TO_EMBED = 5; // Discard chunks shorter than - // this + private int minChunkLengthToEmbed = 5; // Discard chunks shorter than this - private static final int MAX_NUM_CHUNKS = 10000; // The maximum number of chunks to - // generate from a text + private int maxNumChunks = 10000; // The maximum number of chunks to generate from a + // text + + private boolean keepSeparator = true; + + public TokenTextSplitter() { + } + + public TokenTextSplitter(boolean keepSeparator) { + this.keepSeparator = keepSeparator; + } private final EncodingRegistry registry = Encodings.newLazyEncodingRegistry(); @@ -31,7 +39,7 @@ public class TokenTextSplitter extends TextSplitter { @Override protected List splitText(String text) { - return split(text, DEFAULT_CHUNK_SIZE); + return split(text, defaultChunkSize); } public List split(String text, int chunkSize) { @@ -42,7 +50,7 @@ public class TokenTextSplitter extends TextSplitter { List tokens = getEncodedTokens(text); List chunks = new ArrayList<>(); int num_chunks = 0; - while (!tokens.isEmpty() && num_chunks < MAX_NUM_CHUNKS) { + while (!tokens.isEmpty() && num_chunks < maxNumChunks) { List chunk = tokens.subList(0, Math.min(chunkSize, tokens.size())); String chunkText = decodeTokens(chunk); @@ -56,12 +64,13 @@ public class TokenTextSplitter extends TextSplitter { int lastPunctuation = Math.max(chunkText.lastIndexOf('.'), Math.max(chunkText.lastIndexOf('?'), Math.max(chunkText.lastIndexOf('!'), chunkText.lastIndexOf('\n')))); - if (lastPunctuation != -1 && lastPunctuation > MIN_CHUNK_SIZE_CHARS) { + if (lastPunctuation != -1 && lastPunctuation > minChunkSizeChars) { // Truncate the chunk text at the punctuation mark chunkText = chunkText.substring(0, lastPunctuation + 1); } - String chunk_text_to_append = chunkText.replace("\n", " ").trim(); - if (chunk_text_to_append.length() > MIN_CHUNK_LENGTH_TO_EMBED) { + + String chunk_text_to_append = (this.keepSeparator) ? chunkText.trim() : chunkText.replace("\n", " ").trim(); + if (chunk_text_to_append.length() > minChunkLengthToEmbed) { chunks.add(chunk_text_to_append); } @@ -74,7 +83,7 @@ public class TokenTextSplitter extends TextSplitter { // Handle the remaining tokens if (!tokens.isEmpty()) { String remaining_text = decodeTokens(tokens).replace("\n", " ").trim(); - if (remaining_text.length() > MIN_CHUNK_LENGTH_TO_EMBED) { + if (remaining_text.length() > minChunkLengthToEmbed) { chunks.add(remaining_text); } } diff --git a/spring-ai-core/src/test/java/org/springframework/ai/loader/LoaderTests.java b/spring-ai-core/src/test/java/org/springframework/ai/loader/LoaderTests.java index 22e3eaa47..f2ac278c6 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/loader/LoaderTests.java +++ b/spring-ai-core/src/test/java/org/springframework/ai/loader/LoaderTests.java @@ -20,7 +20,7 @@ public class LoaderTests { @Test void loadJson() { assertThat(resource).isNotNull(); - JsonLoader jsonLoader = new JsonLoader("description", resource); + JsonLoader jsonLoader = new JsonLoader(resource, "description"); List documents = jsonLoader.load(); assertThat(documents).isNotEmpty(); for (Document document : documents) { diff --git a/spring-ai-openai/src/main/java/org/springframework/ai/openai/client/OpenAiClient.java b/spring-ai-openai/src/main/java/org/springframework/ai/openai/client/OpenAiClient.java index 1f84279ac..b503dc504 100644 --- a/spring-ai-openai/src/main/java/org/springframework/ai/openai/client/OpenAiClient.java +++ b/spring-ai-openai/src/main/java/org/springframework/ai/openai/client/OpenAiClient.java @@ -41,7 +41,7 @@ public class OpenAiClient implements AiClient { // TODO how to set default options for the entire client // TODO expose request options into Prompt API via PromptOptions - private Double temperature = 0.5; + private Double temperature = 0.7; private String model = "gpt-3.5-turbo"; diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java index 34ccb902d..85c5dd301 100644 --- a/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiTestConfiguration.java @@ -27,6 +27,7 @@ public class OpenAiTestConfiguration { @Bean public OpenAiClient openAiClient(OpenAiService theoOpenAiService) { OpenAiClient openAiClient = new OpenAiClient(theoOpenAiService); + openAiClient.setTemperature(0.3); return openAiClient; } diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIntegrationTest.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIntegrationTest.java index 953fac646..8c383b4f9 100644 --- a/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIntegrationTest.java +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIntegrationTest.java @@ -1,32 +1,42 @@ package org.springframework.ai.openai.acme; import org.junit.jupiter.api.Test; -import org.springframework.ai.document.Document; -import org.springframework.ai.client.AiResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.client.AiClient; +import org.springframework.ai.client.AiResponse; +import org.springframework.ai.document.Document; import org.springframework.ai.loader.impl.JsonLoader; +import org.springframework.ai.openai.embedding.OpenAiEmbeddingClient; +import org.springframework.ai.openai.testutils.AbstractIntegrationTest; import org.springframework.ai.prompt.Prompt; -import org.springframework.ai.prompt.messages.SystemMessage; +import org.springframework.ai.prompt.SystemPromptTemplate; +import org.springframework.ai.prompt.messages.Message; import org.springframework.ai.prompt.messages.UserMessage; import org.springframework.ai.retriever.impl.VectorStoreRetriever; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.ai.vectorstore.impl.InMemoryVectorStore; -import org.springframework.ai.openai.embedding.OpenAiEmbeddingClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.io.Resource; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; @SpringBootTest -public class AcmeIntegrationTest { +public class AcmeIntegrationTest extends AbstractIntegrationTest { - @Value("classpath:bikes.json") - private Resource resource; + private static final Logger logger = LoggerFactory.getLogger(AcmeIntegrationTest.class); + + @Value("classpath:/data/acme/bikes.json") + private Resource bikesResource; + + @Value("classpath:/prompts/acme/system-qa.st") + private Resource systemBikePrompt; @Autowired private OpenAiEmbeddingClient embeddingClient; @@ -36,19 +46,21 @@ public class AcmeIntegrationTest { @Test void beanTest() { - assertThat(resource).isNotNull(); + assertThat(bikesResource).isNotNull(); assertThat(embeddingClient).isNotNull(); assertThat(aiClient).isNotNull(); } + // @Test void acmeChain() { // Step 1 - load documents - JsonLoader jsonLoader = new JsonLoader("description", resource); + JsonLoader jsonLoader = new JsonLoader(bikesResource, "name", "price", "shortDescription", "description"); List documents = jsonLoader.load(); // Step 2 - Create embeddings and save to vector store + logger.info("Creating Embeddings..."); VectorStore vectorStore = new InMemoryVectorStore(embeddingClient); vectorStore.add(documents); @@ -56,38 +68,46 @@ public class AcmeIntegrationTest { // Now user query // This will be wrapped up in a chain - VectorStoreRetriever vectorStoreRetriever = new VectorStoreRetriever(vectorStore); - String userQuery = "What bike is good for city commuting?"; + logger.info("Retrieving relevant documents"); + String userQuery = "How much does the SonicRide 8S cost?"; + // "Tell me about the bike 'The SonicRide 8S'" ; + + // "What bike is good for city commuting?"; List similarDocuments = vectorStoreRetriever.retrieve(userQuery); + logger.info(String.format("Found %s relevant documents.", similarDocuments.size())); // Try the case where not product was specified, so query over whatever docs might // be releveant. - SystemMessage systemMessage = getSystemMessage(similarDocuments); + Message systemMessage = getSystemMessage(similarDocuments); UserMessage userMessage = new UserMessage(userQuery); // Create the prompt ad-hoc for now, need to put in system message and user - // message via ChatPromptTemplate or some other message building mechanic + // message via ChatPromptTemplate or some other message building mechanic; + logger.info("Asking AI model to reply to question."); Prompt prompt = new Prompt(List.of(systemMessage, userMessage)); + logger.info("AI responded."); AiResponse response = aiClient.generate(prompt); + evaluateQuestionAndAnswer(userQuery, response, true); + // Chain // qa = new ConversationalRetrievalChain(llmClient, userPromptTemplate, // vectorStoreRetriever, ) } - private SystemMessage getSystemMessage(List similarDocuments) { + private Message getSystemMessage(List similarDocuments) { // Would need to figure out which of the documenta metadata fields to add, from // the loader, now just the 'full description.' - String systemMessageText = similarDocuments.stream() - .map(entry -> entry.getContent()) - .collect(Collectors.joining("\n")); + String documents = similarDocuments.stream().map(entry -> entry.getContent()).collect(Collectors.joining("\n")); - return new SystemMessage(systemMessageText); + SystemPromptTemplate systemPromptTemplate = new SystemPromptTemplate(systemBikePrompt); + Message systemMessage = systemPromptTemplate.createMessage(Map.of("documents", documents)); + return systemMessage; } diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIntegrationTests.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIntegrationTests.java index bbab2ad83..6a43082f6 100644 --- a/spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIntegrationTests.java +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/client/ClientIntegrationTests.java @@ -1,16 +1,16 @@ package org.springframework.ai.openai.client; import org.junit.jupiter.api.Test; +import org.springframework.ai.client.AiResponse; import org.springframework.ai.client.Generation; +import org.springframework.ai.openai.testutils.AbstractIntegrationTest; import org.springframework.ai.parser.JsonOutputParser; import org.springframework.ai.parser.ListOutputParser; import org.springframework.ai.prompt.Prompt; import org.springframework.ai.prompt.PromptTemplate; import org.springframework.ai.prompt.SystemPromptTemplate; import org.springframework.ai.prompt.messages.Message; -import org.springframework.ai.prompt.messages.SystemMessage; import org.springframework.ai.prompt.messages.UserMessage; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.convert.support.DefaultConversionService; @@ -24,20 +24,11 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @SpringBootTest -class ClientIntegrationTests { - - @Autowired - OpenAiClient openAiClient; +class ClientIntegrationTests extends AbstractIntegrationTest { @Value("classpath:/prompts/system-message.st") private Resource systemResource; - @Value("classpath:/prompts/system-evaluator-message.st") - private Resource systemEvaluatorResource; - - @Value("classpath:/prompts/user-evaluator-message.st") - private Resource userEvaluatorResource; - @Test void roleTest() { String request = "Tell me about 3 famous pirates from the Golden Age of Piracy and why they did."; @@ -47,22 +38,8 @@ class ClientIntegrationTests { SystemPromptTemplate systemPromptTemplate = new SystemPromptTemplate(systemResource); Message systemMessage = systemPromptTemplate.createMessage(Map.of("name", name, "voice", voice)); Prompt prompt = new Prompt(List.of(userMessage, systemMessage)); - Generation response = openAiClient.generate(prompt).getGeneration(); - System.out.println(response); - assertThat(response).isNotNull(); - - evaluateQuestionAndAnswer(request, response.getText()); - } - - private void evaluateQuestionAndAnswer(String question, String answer) { - PromptTemplate userPromptTemplate = new PromptTemplate(userEvaluatorResource, - Map.of("question", question, "answer", answer)); - SystemMessage systemMessage = new SystemMessage(systemEvaluatorResource); - Message userMessage = userPromptTemplate.createMessage(); - Prompt prompt = new Prompt(List.of(userMessage, systemMessage)); - Generation response = openAiClient.generate(prompt).getGeneration(); - System.out.println(response); - assertThat(response.getText()).isEqualTo("YES"); + AiResponse response = openAiClient.generate(prompt); + evaluateQuestionAndAnswer(request, response, false); } @Test diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIntegrationTest.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIntegrationTest.java new file mode 100644 index 000000000..0d3bd07ff --- /dev/null +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/testutils/AbstractIntegrationTest.java @@ -0,0 +1,70 @@ +package org.springframework.ai.openai.testutils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.client.AiResponse; +import org.springframework.ai.openai.client.OpenAiClient; +import org.springframework.ai.prompt.Prompt; +import org.springframework.ai.prompt.PromptTemplate; +import org.springframework.ai.prompt.messages.Message; +import org.springframework.ai.prompt.messages.SystemMessage; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class AbstractIntegrationTest { + + private static final Logger logger = LoggerFactory.getLogger(AbstractIntegrationTest.class); + + @Autowired + protected OpenAiClient openAiClient; + + @Value("classpath:/prompts/eval/qa-evaluator-accurate-answer.st") + protected Resource qaEvaluatorAccurateAnswerResource; + + @Value("classpath:/prompts/eval/qa-evaluator-not-related-message.st") + protected Resource qaEvaluatorNotRelatedResource; + + @Value("classpath:/prompts/eval/qa-evaluator-fact-based-answer.st") + protected Resource qaEvalutaorFactBasedAnswerResource; + + @Value("classpath:/prompts/eval/user-evaluator-message.st") + protected Resource userEvaluatorResource; + + protected void evaluateQuestionAndAnswer(String question, AiResponse response, boolean factBased) { + assertThat(response).isNotNull(); + String answer = response.getGeneration().getText(); + logger.info("Question: " + question); + logger.info("Answer:" + answer); + PromptTemplate userPromptTemplate = new PromptTemplate(userEvaluatorResource, + Map.of("question", question, "answer", answer)); + SystemMessage systemMessage; + if (factBased) { + systemMessage = new SystemMessage(qaEvalutaorFactBasedAnswerResource); + } + else { + systemMessage = new SystemMessage(qaEvaluatorAccurateAnswerResource); + } + Message userMessage = userPromptTemplate.createMessage(); + Prompt prompt = new Prompt(List.of(userMessage, systemMessage)); + String yesOrNo = openAiClient.generate(prompt).getGeneration().getText(); + logger.info("Is Answer related to question: " + yesOrNo); + if (yesOrNo.equalsIgnoreCase("no")) { + SystemMessage notRelatedSystemMessage = new SystemMessage(qaEvaluatorNotRelatedResource); + prompt = new Prompt(List.of(userMessage, notRelatedSystemMessage)); + String reasonForFailure = openAiClient.generate(prompt).getGeneration().getText(); + fail(reasonForFailure); + } + else { + logger.info("Answer is related to question."); + assertThat(yesOrNo).isEqualTo("YES"); + } + } + +} diff --git a/spring-ai-openai/src/test/resources/bikes.json b/spring-ai-openai/src/test/resources/data/acme/bikes.json similarity index 100% rename from spring-ai-openai/src/test/resources/bikes.json rename to spring-ai-openai/src/test/resources/data/acme/bikes.json diff --git a/spring-ai-openai/src/test/resources/prompts/acme/system-qa.st b/spring-ai-openai/src/test/resources/prompts/acme/system-qa.st new file mode 100644 index 000000000..44db6f210 --- /dev/null +++ b/spring-ai-openai/src/test/resources/prompts/acme/system-qa.st @@ -0,0 +1,7 @@ +You're assisting with questions about products in a bicycle catalog. +Use the information from the DOCUMENTS section to provide accurate answers. +The the answer involves referring to the price or the dimension of the bicycle, include the bicycle name in the response. +If unsure, simply state that you don't know. + +DOCUMENTS: +{documents} \ No newline at end of file diff --git a/spring-ai-openai/src/test/resources/prompts/system-evaluator-message.st b/spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-accurate-answer.st similarity index 100% rename from spring-ai-openai/src/test/resources/prompts/system-evaluator-message.st rename to spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-accurate-answer.st diff --git a/spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-fact-based-answer.st b/spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-fact-based-answer.st new file mode 100644 index 000000000..22fc3e88d --- /dev/null +++ b/spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-fact-based-answer.st @@ -0,0 +1,7 @@ +You are an AI evaluator. Your task is to verify if the provided ANSWER is a direct and accurate response to the given QUESTION. If the ANSWER is correct and directly answers the QUESTION, reply with "YES". If the ANSWER is not a direct response or is inaccurate, reply with "NO". + +For example: + +If the QUESTION is "What is the capital of France?" and the ANSWER is "Paris.", you should respond with "YES". +If the QUESTION is "What is the capital of France?" and the ANSWER is "France is in Europe.", respond with "NO". +Now, evaluate the following: diff --git a/spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-not-related-message.st b/spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-not-related-message.st new file mode 100644 index 000000000..7c33e675e --- /dev/null +++ b/spring-ai-openai/src/test/resources/prompts/eval/qa-evaluator-not-related-message.st @@ -0,0 +1,4 @@ +You are an AI assistant who helps users to evaluate if the answers to questions are accurate. +You will be provided with a QUESTION and an ANSWER. +A previous evaluation has determined that QUESTION and ANSWER are not related. +Give an explanation as to why they are not related. \ No newline at end of file diff --git a/spring-ai-openai/src/test/resources/prompts/user-evaluator-message.st b/spring-ai-openai/src/test/resources/prompts/eval/user-evaluator-message.st similarity index 100% rename from spring-ai-openai/src/test/resources/prompts/user-evaluator-message.st rename to spring-ai-openai/src/test/resources/prompts/eval/user-evaluator-message.st