From fef730301caa0c437d630d73edcf03275ded4cda Mon Sep 17 00:00:00 2001 From: Christian Tzolov Date: Fri, 8 Mar 2024 15:15:01 +0100 Subject: [PATCH] Replace %n by System.lineSeparator() --- .../ai/bedrock/MessageToPromptConverter.java | 3 +- .../splitter/TokenTextSplitter.java | 36 ++++++++++++------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/MessageToPromptConverter.java b/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/MessageToPromptConverter.java index 0153b9046..d20041541 100644 --- a/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/MessageToPromptConverter.java +++ b/models/spring-ai-bedrock/src/main/java/org/springframework/ai/bedrock/MessageToPromptConverter.java @@ -67,7 +67,8 @@ public class MessageToPromptConverter { .map(this::messageToString) .collect(Collectors.joining(System.lineSeparator())); - final String prompt = String.format("%s%n%n%s%n%s", systemMessages, userMessages, ASSISTANT_PROMPT); + final String prompt = systemMessages + System.lineSeparator() + System.lineSeparator() + userMessages + + System.lineSeparator() + ASSISTANT_PROMPT; return prompt; } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java b/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java index c816c00fa..c820b51a3 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java @@ -32,16 +32,21 @@ import org.springframework.util.Assert; */ public class TokenTextSplitter extends TextSplitter { - private final int defaultChunkSize = 800; // The target size of each text - // chunk in tokens + private final EncodingRegistry registry = Encodings.newLazyEncodingRegistry(); - private int minChunkSizeChars = 350; // The minimum size of each text - // chunk in characters + private final Encoding encoding = registry.getEncoding(EncodingType.CL100K_BASE); - private int minChunkLengthToEmbed = 5; // Discard chunks shorter than this + // The target size of each text chunk in tokens + private int defaultChunkSize = 800; - private int maxNumChunks = 10000; // The maximum number of chunks to generate from a - // text + // The minimum size of each text chunk in characters + private int minChunkSizeChars = 350; + + // Discard chunks shorter than this + private int minChunkLengthToEmbed = 5; + + // The maximum number of chunks to generate from a text + private int maxNumChunks = 10000; private boolean keepSeparator = true; @@ -52,9 +57,14 @@ public class TokenTextSplitter extends TextSplitter { this.keepSeparator = keepSeparator; } - private final EncodingRegistry registry = Encodings.newLazyEncodingRegistry(); - - private final Encoding encoding = registry.getEncoding(EncodingType.CL100K_BASE); + public TokenTextSplitter(int defaultChunkSize, int minChunkSizeChars, int minChunkLengthToEmbed, int maxNumChunks, + boolean keepSeparator) { + this.defaultChunkSize = defaultChunkSize; + this.minChunkSizeChars = minChunkSizeChars; + this.minChunkLengthToEmbed = minChunkLengthToEmbed; + this.maxNumChunks = maxNumChunks; + this.keepSeparator = keepSeparator; + } @Override protected List splitText(String text) { @@ -88,10 +98,10 @@ public class TokenTextSplitter extends TextSplitter { chunkText = chunkText.substring(0, lastPunctuation + 1); } - String chunk_text_to_append = (this.keepSeparator) ? chunkText.trim() + String chunkTextToAppend = (this.keepSeparator) ? chunkText.trim() : chunkText.replace(System.lineSeparator(), " ").trim(); - if (chunk_text_to_append.length() > this.minChunkLengthToEmbed) { - chunks.add(chunk_text_to_append); + if (chunkTextToAppend.length() > this.minChunkLengthToEmbed) { + chunks.add(chunkTextToAppend); } // Remove the tokens corresponding to the chunk text from the remaining tokens