Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update 3rd party POM dependecies and resolve related issues

Browse Source
This commit is contained in:
Christian Tzolov
2024-02-23 12:43:30 +01:00
parent b2d04a6e1e
commit 56c6abfc9c
3 changed files with 54 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pom.xml
View File

@@ -97,13 +97,13 @@
<maven.compiler.target>17</maven.compiler.target>
<!-- production dependencies -->
<spring-boot.version>3.2.2</spring-boot.version>
<spring-framework.version>6.1.3</spring-framework.version>
<spring-boot.version>3.2.3</spring-boot.version>
<spring-framework.version>6.1.4</spring-framework.version>
<stringtemplate.version>4.0.2</stringtemplate.version>
<azure-open-ai-client.version>1.0.0-beta.6</azure-open-ai-client.version>
<jtokkit.version>0.6.1</jtokkit.version>
<jtokkit.version>1.0.0</jtokkit.version>
<victools.version>4.31.1</victools.version>
<bedrockruntime.version>2.23.10</bedrockruntime.version>
<bedrockruntime.version>2.24.8</bedrockruntime.version>
<jackson.version>2.16.1</jackson.version>
<djl.version>0.26.0</djl.version>
<onnxruntime.version>1.17.0</onnxruntime.version>
@@ -111,16 +111,16 @@
<!-- readers/writer/stores dependencies-->
<pdfbox.version>3.0.1</pdfbox.version>
<pgvector.version>0.1.4</pgvector.version>
<postgresql.version>42.7.1</postgresql.version>
<postgresql.version>42.7.2</postgresql.version>
<milvus.version>2.3.4</milvus.version>
<pinecone.version>0.7.4</pinecone.version>
<protobuf-java-util.version>3.24.4</protobuf-java-util.version>
<fastjson.version>2.0.42</fastjson.version>
<fastjson.version>2.0.46</fastjson.version>
<azure-search.version>11.6.1</azure-search.version>
<weaviate-client.version>4.5.1</weaviate-client.version>
<!-- testing dependencies -->
<testcontainers.version>1.19.0</testcontainers.version>
<testcontainers.version>1.19.6</testcontainers.version>
<!-- documentation dependencies -->
<io.spring.maven.antora-version>0.0.4</io.spring.maven.antora-version>

15
spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TextSplitter.java
View File

@@ -1,3 +1,18 @@
/*
* Copyright 2023-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.transformer.splitter;
import org.slf4j.Logger;

41
spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java
View File

@@ -1,15 +1,34 @@
/*
* Copyright 2023-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.transformer.splitter;
import java.util.ArrayList;
import java.util.List;
import com.knuddels.jtokkit.Encodings;
import com.knuddels.jtokkit.api.Encoding;
import com.knuddels.jtokkit.api.EncodingRegistry;
import com.knuddels.jtokkit.api.EncodingType;
import com.knuddels.jtokkit.api.IntArrayList;
import java.util.ArrayList;
import java.util.List;
import org.springframework.util.Assert;
/**
* @author Raphael Yu
* @author Christian Tzolov
*/
public class TokenTextSplitter extends TextSplitter {
@@ -39,7 +58,7 @@ public class TokenTextSplitter extends TextSplitter {
@Override
protected List<String> splitText(String text) {
return split(text, defaultChunkSize);
return split(text, this.defaultChunkSize);
}
public List<String> split(String text, int chunkSize) {
@@ -50,7 +69,7 @@ public class TokenTextSplitter extends TextSplitter {
List<Integer> tokens = getEncodedTokens(text);
List<String> chunks = new ArrayList<>();
int num_chunks = 0;
while (!tokens.isEmpty() && num_chunks < maxNumChunks) {
while (!tokens.isEmpty() && num_chunks < this.maxNumChunks) {
List<Integer> chunk = tokens.subList(0, Math.min(chunkSize, tokens.size()));
String chunkText = decodeTokens(chunk);
@@ -64,13 +83,13 @@ public class TokenTextSplitter extends TextSplitter {
int lastPunctuation = Math.max(chunkText.lastIndexOf('.'), Math.max(chunkText.lastIndexOf('?'),
Math.max(chunkText.lastIndexOf('!'), chunkText.lastIndexOf('\n'))));
if (lastPunctuation != -1 && lastPunctuation > minChunkSizeChars) {
if (lastPunctuation != -1 && lastPunctuation > this.minChunkSizeChars) {
// Truncate the chunk text at the punctuation mark
chunkText = chunkText.substring(0, lastPunctuation + 1);
}
String chunk_text_to_append = (this.keepSeparator) ? chunkText.trim() : chunkText.replace("\n", " ").trim();
if (chunk_text_to_append.length() > minChunkLengthToEmbed) {
if (chunk_text_to_append.length() > this.minChunkLengthToEmbed) {
chunks.add(chunk_text_to_append);
}
@@ -83,7 +102,7 @@ public class TokenTextSplitter extends TextSplitter {
// Handle the remaining tokens
if (!tokens.isEmpty()) {
String remaining_text = decodeTokens(tokens).replace("\n", " ").trim();
if (remaining_text.length() > minChunkLengthToEmbed) {
if (remaining_text.length() > this.minChunkLengthToEmbed) {
chunks.add(remaining_text);
}
}
@@ -92,11 +111,15 @@ public class TokenTextSplitter extends TextSplitter {
}
private List<Integer> getEncodedTokens(String text) {
return encoding.encode(text);
Assert.notNull(text, "Text must not be null");
return this.encoding.encode(text).boxed();
}
private String decodeTokens(List<Integer> tokens) {
return encoding.decode(tokens);
Assert.notNull(tokens, "Tokens must not be null");
var tokensIntArray = new IntArrayList(tokens.size());
tokens.forEach(tokensIntArray::add);
return this.encoding.decode(tokensIntArray);
}
}