From 3eff6a75c928ddfbd96dcb35154739aba98c84fb Mon Sep 17 00:00:00 2001 From: Christian Tzolov Date: Tue, 3 Oct 2023 21:10:56 +0200 Subject: [PATCH] Refactoring to functional interfaces Fixes #47 reformatting --- .../ai/document/DocumentReader.java | 8 +++ .../ai/document/DocumentRetriever.java | 19 +++++++ .../ai/document/DocumentWriter.java | 27 ++++++++++ .../org/springframework/ai/loader/Loader.java | 14 ------ .../AbstractMetadataFeatureExtractor.java | 49 ------------------- .../EmptyJsonMetadataGenerator.java | 2 +- .../JsonMetadataGenerator.java | 2 +- .../JsonReader.java} | 43 ++++++---------- .../TextReader.java} | 24 ++++----- .../ai/retriever/Retriever.java | 16 ------ .../{impl => }/VectorStoreRetriever.java | 6 +-- .../ContentFormatTransformer.java} | 8 +-- .../KeywordMetadataEnricher.java} | 6 +-- .../SummaryMetadataEnricher.java} | 8 +-- .../splitter/TextSplitter.java | 2 +- .../splitter/TokenTextSplitter.java | 2 +- .../{impl => }/InMemoryVectorStore.java | 3 +- .../SimplePersistentVectorStore.java | 2 +- .../ai/vectorstore/VectorStore.java | 7 ++- .../{loader => reader}/JsonLoaderTests.java | 8 +-- .../{loader => reader}/TextLoaderTests.java | 15 +++--- .../splitter/TextSplitterTests.java | 3 +- .../ai/openai/acme/AcmeIT.java | 16 +++--- .../MetadataTransformerIT.java} | 37 +++++++------- .../SimplePersistentVectorStoreIT.java | 10 ++-- 25 files changed, 153 insertions(+), 184 deletions(-) create mode 100644 spring-ai-core/src/main/java/org/springframework/ai/document/DocumentReader.java create mode 100644 spring-ai-core/src/main/java/org/springframework/ai/document/DocumentRetriever.java create mode 100644 spring-ai-core/src/main/java/org/springframework/ai/document/DocumentWriter.java delete mode 100644 spring-ai-core/src/main/java/org/springframework/ai/loader/Loader.java delete mode 100644 spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/AbstractMetadataFeatureExtractor.java rename spring-ai-core/src/main/java/org/springframework/ai/{loader/impl => reader}/EmptyJsonMetadataGenerator.java (87%) rename spring-ai-core/src/main/java/org/springframework/ai/{loader/impl => reader}/JsonMetadataGenerator.java (87%) rename spring-ai-core/src/main/java/org/springframework/ai/{loader/impl/JsonLoader.java => reader/JsonReader.java} (65%) rename spring-ai-core/src/main/java/org/springframework/ai/{loader/impl/TextLoader.java => reader/TextReader.java} (75%) delete mode 100644 spring-ai-core/src/main/java/org/springframework/ai/retriever/Retriever.java rename spring-ai-core/src/main/java/org/springframework/ai/retriever/{impl => }/VectorStoreRetriever.java (88%) rename spring-ai-core/src/main/java/org/springframework/ai/{loader/extractor/ContentFormatEnricher.java => transformer/ContentFormatTransformer.java} (90%) rename spring-ai-core/src/main/java/org/springframework/ai/{loader/extractor/KeywordExtractor.java => transformer/KeywordMetadataEnricher.java} (92%) rename spring-ai-core/src/main/java/org/springframework/ai/{loader/extractor/SummaryExtractor.java => transformer/SummaryMetadataEnricher.java} (92%) rename spring-ai-core/src/main/java/org/springframework/ai/{ => transformer}/splitter/TextSplitter.java (97%) rename spring-ai-core/src/main/java/org/springframework/ai/{ => transformer}/splitter/TokenTextSplitter.java (98%) rename spring-ai-core/src/main/java/org/springframework/ai/vectorstore/{impl => }/InMemoryVectorStore.java (97%) rename spring-ai-core/src/main/java/org/springframework/ai/vectorstore/{impl => }/SimplePersistentVectorStore.java (98%) rename spring-ai-core/src/test/java/org/springframework/ai/{loader => reader}/JsonLoaderTests.java (86%) rename spring-ai-core/src/test/java/org/springframework/ai/{loader => reader}/TextLoaderTests.java (76%) rename spring-ai-core/src/test/java/org/springframework/ai/{ => transformer}/splitter/TextSplitterTests.java (97%) rename spring-ai-openai/src/test/java/org/springframework/ai/openai/{extractor/MetadataExtractorIT.java => transformer/MetadataTransformerIT.java} (84%) diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentReader.java b/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentReader.java new file mode 100644 index 000000000..f46171d33 --- /dev/null +++ b/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentReader.java @@ -0,0 +1,8 @@ +package org.springframework.ai.document; + +import java.util.List; +import java.util.function.Supplier; + +public interface DocumentReader extends Supplier> { + +} diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentRetriever.java b/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentRetriever.java new file mode 100644 index 000000000..bcab24fc8 --- /dev/null +++ b/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentRetriever.java @@ -0,0 +1,19 @@ +package org.springframework.ai.document; + +import java.util.List; +import java.util.function.Function; + +public interface DocumentRetriever extends Function> { + + /** + * Retrieves relevant documents however the implementation sees fit. + * @param query query string + * @return relevant documents + */ + List retrieve(String query); + + default List apply(String query) { + return retrieve(query); + } + +} diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentWriter.java b/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentWriter.java new file mode 100644 index 000000000..69ded8404 --- /dev/null +++ b/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentWriter.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.document; + +import java.util.List; +import java.util.function.Consumer; + +/** + * @author Christian Tzolov + */ +public interface DocumentWriter extends Consumer> { + +} diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/Loader.java b/spring-ai-core/src/main/java/org/springframework/ai/loader/Loader.java deleted file mode 100644 index af5970c4c..000000000 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/Loader.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.springframework.ai.loader; - -import org.springframework.ai.document.Document; -import org.springframework.ai.splitter.TextSplitter; - -import java.util.List; - -public interface Loader { - - List load(); - - List load(TextSplitter textSplitter); - -} diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/AbstractMetadataFeatureExtractor.java b/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/AbstractMetadataFeatureExtractor.java deleted file mode 100644 index 6646560fd..000000000 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/AbstractMetadataFeatureExtractor.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.ai.loader.extractor; - -import java.util.List; -import java.util.Map; - -import org.springframework.ai.document.Document; -import org.springframework.ai.document.DocumentTransformer; - -/** - * @author Christian Tzolov - */ -public abstract class AbstractMetadataFeatureExtractor implements DocumentTransformer { - - @Override - public List apply(List documents) { - List> metadataList = this.extract(documents); - - for (int idx = 0; idx < documents.size(); idx++) { - documents.get(idx).getMetadata().putAll(metadataList.get(idx)); - } - - return documents; - } - - /** - * Extracts metadata for a list of documents, returning a list of metadata - * dictionaries corresponding to each document. - * @param documents Documents to extract metadata from. - * @return List of metadata dictionaries corresponding to each document - */ - abstract public List> extract(List documents); - -} diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/EmptyJsonMetadataGenerator.java b/spring-ai-core/src/main/java/org/springframework/ai/reader/EmptyJsonMetadataGenerator.java similarity index 87% rename from spring-ai-core/src/main/java/org/springframework/ai/loader/impl/EmptyJsonMetadataGenerator.java rename to spring-ai-core/src/main/java/org/springframework/ai/reader/EmptyJsonMetadataGenerator.java index 948f53895..2218159b8 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/EmptyJsonMetadataGenerator.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/reader/EmptyJsonMetadataGenerator.java @@ -1,4 +1,4 @@ -package org.springframework.ai.loader.impl; +package org.springframework.ai.reader; import java.util.Collections; import java.util.Map; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonMetadataGenerator.java b/spring-ai-core/src/main/java/org/springframework/ai/reader/JsonMetadataGenerator.java similarity index 87% rename from spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonMetadataGenerator.java rename to spring-ai-core/src/main/java/org/springframework/ai/reader/JsonMetadataGenerator.java index 61774f6e6..68721d84c 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonMetadataGenerator.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/reader/JsonMetadataGenerator.java @@ -1,4 +1,4 @@ -package org.springframework.ai.loader.impl; +package org.springframework.ai.reader; import java.util.Map; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonLoader.java b/spring-ai-core/src/main/java/org/springframework/ai/reader/JsonReader.java similarity index 65% rename from spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonLoader.java rename to spring-ai-core/src/main/java/org/springframework/ai/reader/JsonReader.java index 48ad2b4ea..7195a3789 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/JsonLoader.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/reader/JsonReader.java @@ -1,17 +1,19 @@ -package org.springframework.ai.loader.impl; +package org.springframework.ai.reader; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; + import org.springframework.ai.document.Document; -import org.springframework.ai.loader.Loader; -import org.springframework.ai.splitter.TextSplitter; -import org.springframework.ai.splitter.TokenTextSplitter; +import org.springframework.ai.document.DocumentReader; import org.springframework.core.io.Resource; -import java.io.IOException; -import java.util.*; - -public class JsonLoader implements Loader { +public class JsonReader implements DocumentReader { private Resource resource; @@ -22,15 +24,15 @@ public class JsonLoader implements Loader { */ private List jsonKeysToUse; - public JsonLoader(Resource resource) { + public JsonReader(Resource resource) { this(resource, new ArrayList<>().toArray(new String[0])); } - public JsonLoader(Resource resource, String... jsonKeysToUse) { + public JsonReader(Resource resource, String... jsonKeysToUse) { this(resource, new EmptyJsonMetadataGenerator(), jsonKeysToUse); } - public JsonLoader(Resource resource, JsonMetadataGenerator jsonMetadataGenerator, String... jsonKeysToUse) { + public JsonReader(Resource resource, JsonMetadataGenerator jsonMetadataGenerator, String... jsonKeysToUse) { Objects.requireNonNull(jsonKeysToUse, "keys must not be null"); Objects.requireNonNull(jsonMetadataGenerator, "jsonMetadataGenerator must not be null"); Objects.requireNonNull(resource, "The Spring Resource must not be null"); @@ -40,13 +42,7 @@ public class JsonLoader implements Loader { } @Override - public List load() { - return load(new TokenTextSplitter()); - } - - @Override - public List load(TextSplitter textSplitter) { - + public List get() { ObjectMapper objectMapper = new ObjectMapper(); List documents = new ArrayList<>(); try { @@ -74,15 +70,8 @@ public class JsonLoader implements Loader { else { document = new Document(item.toString(), metadata); } - // Splitting at the item level is good when the size of the json per - // element is large - // as is the case with a catalog of product, as the metadata applies - // across all split documents - // This may not be good when the size of the json element is small as it - // can create too many individual - // documents. - List splitDocuments = textSplitter.apply(List.of(document)); - documents.addAll(splitDocuments); + + documents.add(document); } } catch (IOException e) { diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/TextLoader.java b/spring-ai-core/src/main/java/org/springframework/ai/reader/TextReader.java similarity index 75% rename from spring-ai-core/src/main/java/org/springframework/ai/loader/impl/TextLoader.java rename to spring-ai-core/src/main/java/org/springframework/ai/reader/TextReader.java index 806251a42..42355fca6 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/impl/TextLoader.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/reader/TextReader.java @@ -1,18 +1,15 @@ -package org.springframework.ai.loader.impl; +package org.springframework.ai.reader; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import org.springframework.ai.document.Document; -import org.springframework.ai.loader.Loader; -import org.springframework.ai.splitter.TextSplitter; -import org.springframework.ai.splitter.TokenTextSplitter; +import org.springframework.ai.document.DocumentReader; import org.springframework.core.io.DefaultResourceLoader; import org.springframework.core.io.Resource; import org.springframework.util.StreamUtils; @@ -21,7 +18,7 @@ import org.springframework.util.StreamUtils; * @author Craig Walls * @author Christian Tzolov */ -public class TextLoader implements Loader { +public class TextReader implements DocumentReader { public static final String CHARSET_METADATA = "charset"; @@ -39,11 +36,11 @@ public class TextLoader implements Loader { private Map customMetadata = new HashMap<>(); - public TextLoader(String resourceUrl) { + public TextReader(String resourceUrl) { this(new DefaultResourceLoader().getResource(resourceUrl)); } - public TextLoader(Resource resource) { + public TextReader(Resource resource) { Objects.requireNonNull(resource, "The Spring Resource must not be null"); this.resource = resource; } @@ -66,12 +63,7 @@ public class TextLoader implements Loader { } @Override - public List load() { - return load(new TokenTextSplitter()); - } - - @Override - public List load(TextSplitter textSplitter) { + public List get() { try { String document = StreamUtils.copyToString(this.resource.getInputStream(), this.charset); @@ -80,7 +72,9 @@ public class TextLoader implements Loader { this.customMetadata.put(CHARSET_METADATA, this.charset.name()); this.customMetadata.put(SOURCE_METADATA, this.resource.getFilename()); - return textSplitter.apply(Collections.singletonList(new Document(document, this.customMetadata))); + return List.of(new Document(document, this.customMetadata)); + // return textSplitter.apply(Collections.singletonList(new Document(document, + // this.customMetadata))); } catch (IOException e) { throw new RuntimeException(e); diff --git a/spring-ai-core/src/main/java/org/springframework/ai/retriever/Retriever.java b/spring-ai-core/src/main/java/org/springframework/ai/retriever/Retriever.java deleted file mode 100644 index c50203735..000000000 --- a/spring-ai-core/src/main/java/org/springframework/ai/retriever/Retriever.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.springframework.ai.retriever; - -import org.springframework.ai.document.Document; - -import java.util.List; - -public interface Retriever { - - /** - * Retrieves relevant documents however the implementation sees fit. - * @param query query string - * @return relevant documents - */ - List retrieve(String query); - -} diff --git a/spring-ai-core/src/main/java/org/springframework/ai/retriever/impl/VectorStoreRetriever.java b/spring-ai-core/src/main/java/org/springframework/ai/retriever/VectorStoreRetriever.java similarity index 88% rename from spring-ai-core/src/main/java/org/springframework/ai/retriever/impl/VectorStoreRetriever.java rename to spring-ai-core/src/main/java/org/springframework/ai/retriever/VectorStoreRetriever.java index aba4b3881..7c416216c 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/retriever/impl/VectorStoreRetriever.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/retriever/VectorStoreRetriever.java @@ -1,14 +1,14 @@ -package org.springframework.ai.retriever.impl; +package org.springframework.ai.retriever; import org.springframework.ai.document.Document; -import org.springframework.ai.retriever.Retriever; +import org.springframework.ai.document.DocumentRetriever; import org.springframework.ai.vectorstore.VectorStore; import java.util.List; import java.util.Objects; import java.util.Optional; -public class VectorStoreRetriever implements Retriever { +public class VectorStoreRetriever implements DocumentRetriever { private VectorStore vectorStore; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/ContentFormatEnricher.java b/spring-ai-core/src/main/java/org/springframework/ai/transformer/ContentFormatTransformer.java similarity index 90% rename from spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/ContentFormatEnricher.java rename to spring-ai-core/src/main/java/org/springframework/ai/transformer/ContentFormatTransformer.java index fcbaee0b5..4aa61838b 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/ContentFormatEnricher.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/transformer/ContentFormatTransformer.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.loader.extractor; +package org.springframework.ai.transformer; import java.util.ArrayList; import java.util.List; @@ -27,7 +27,7 @@ import org.springframework.ai.document.DocumentTransformer; /** * @author Christian Tzolov */ -public class ContentFormatEnricher implements DocumentTransformer { +public class ContentFormatTransformer implements DocumentTransformer { /** * Disable the content-formatter template rewrite. @@ -36,11 +36,11 @@ public class ContentFormatEnricher implements DocumentTransformer { private ContentFormatter contentFormatter; - public ContentFormatEnricher(ContentFormatter contentFormatter) { + public ContentFormatTransformer(ContentFormatter contentFormatter) { this(contentFormatter, false); } - public ContentFormatEnricher(ContentFormatter contentFormatter, boolean disableTemplateRewrite) { + public ContentFormatTransformer(ContentFormatter contentFormatter, boolean disableTemplateRewrite) { this.contentFormatter = contentFormatter; this.disableTemplateRewrite = disableTemplateRewrite; } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/KeywordExtractor.java b/spring-ai-core/src/main/java/org/springframework/ai/transformer/KeywordMetadataEnricher.java similarity index 92% rename from spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/KeywordExtractor.java rename to spring-ai-core/src/main/java/org/springframework/ai/transformer/KeywordMetadataEnricher.java index 5c9b82917..6496c666b 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/KeywordExtractor.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/transformer/KeywordMetadataEnricher.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.loader.extractor; +package org.springframework.ai.transformer; import java.util.List; import java.util.Map; @@ -31,7 +31,7 @@ import org.springframework.util.Assert; * * @author Christian Tzolov */ -public class KeywordExtractor implements DocumentTransformer { +public class KeywordMetadataEnricher implements DocumentTransformer { private static final String EXCERPT_KEYWORDS_METADATA_KEY = "excerpt_keywords"; @@ -51,7 +51,7 @@ public class KeywordExtractor implements DocumentTransformer { */ private final int keywordCount; - public KeywordExtractor(AiClient aiClient, int keywordCount) { + public KeywordMetadataEnricher(AiClient aiClient, int keywordCount) { Assert.notNull(aiClient, "AiClient must not be null"); Assert.isTrue(keywordCount >= 1, "Document count must be >= 1"); diff --git a/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/SummaryExtractor.java b/spring-ai-core/src/main/java/org/springframework/ai/transformer/SummaryMetadataEnricher.java similarity index 92% rename from spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/SummaryExtractor.java rename to spring-ai-core/src/main/java/org/springframework/ai/transformer/SummaryMetadataEnricher.java index 3521e7a73..3ceb62244 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/loader/extractor/SummaryExtractor.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/transformer/SummaryMetadataEnricher.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.loader.extractor; +package org.springframework.ai.transformer; import java.util.ArrayList; import java.util.HashMap; @@ -36,7 +36,7 @@ import org.springframework.util.CollectionUtils; * * @author Christian Tzolov */ -public class SummaryExtractor implements DocumentTransformer { +public class SummaryMetadataEnricher implements DocumentTransformer { private static final String SECTION_SUMMARY_METADATA_KEY = "section_summary"; @@ -77,11 +77,11 @@ public class SummaryExtractor implements DocumentTransformer { */ private final String summaryTemplate; - public SummaryExtractor(AiClient aiClient, List summaryTypes) { + public SummaryMetadataEnricher(AiClient aiClient, List summaryTypes) { this(aiClient, summaryTypes, DEFAULT_SUMMARY_EXTRACT_TEMPLATE, MetadataMode.ALL); } - public SummaryExtractor(AiClient aiClient, List summaryTypes, String summaryTemplate, + public SummaryMetadataEnricher(AiClient aiClient, List summaryTypes, String summaryTemplate, MetadataMode metadataMode) { Assert.notNull(aiClient, "AiClient must not be null"); Assert.hasText(summaryTemplate, "Summary template must not be empty"); diff --git a/spring-ai-core/src/main/java/org/springframework/ai/splitter/TextSplitter.java b/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TextSplitter.java similarity index 97% rename from spring-ai-core/src/main/java/org/springframework/ai/splitter/TextSplitter.java rename to spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TextSplitter.java index bdf4b4618..82526fe31 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/splitter/TextSplitter.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TextSplitter.java @@ -1,4 +1,4 @@ -package org.springframework.ai.splitter; +package org.springframework.ai.transformer.splitter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/splitter/TokenTextSplitter.java b/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java similarity index 98% rename from spring-ai-core/src/main/java/org/springframework/ai/splitter/TokenTextSplitter.java rename to spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java index d2d92cbbc..52204931b 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/splitter/TokenTextSplitter.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java @@ -1,4 +1,4 @@ -package org.springframework.ai.splitter; +package org.springframework.ai.transformer.splitter; import com.knuddels.jtokkit.Encodings; import com.knuddels.jtokkit.api.Encoding; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/impl/InMemoryVectorStore.java b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/InMemoryVectorStore.java similarity index 97% rename from spring-ai-core/src/main/java/org/springframework/ai/vectorstore/impl/InMemoryVectorStore.java rename to spring-ai-core/src/main/java/org/springframework/ai/vectorstore/InMemoryVectorStore.java index e7852aea9..2d6c0531f 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/impl/InMemoryVectorStore.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/InMemoryVectorStore.java @@ -1,10 +1,9 @@ -package org.springframework.ai.vectorstore.impl; +package org.springframework.ai.vectorstore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingClient; -import org.springframework.ai.vectorstore.VectorStore; import java.util.*; import java.util.concurrent.ConcurrentHashMap; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/impl/SimplePersistentVectorStore.java b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/SimplePersistentVectorStore.java similarity index 98% rename from spring-ai-core/src/main/java/org/springframework/ai/vectorstore/impl/SimplePersistentVectorStore.java rename to spring-ai-core/src/main/java/org/springframework/ai/vectorstore/SimplePersistentVectorStore.java index bfd70363f..81b39e21d 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/impl/SimplePersistentVectorStore.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/SimplePersistentVectorStore.java @@ -1,4 +1,4 @@ -package org.springframework.ai.vectorstore.impl; +package org.springframework.ai.vectorstore; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/VectorStore.java b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/VectorStore.java index df78cef41..2b3047a61 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/VectorStore.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/VectorStore.java @@ -4,8 +4,9 @@ import java.util.List; import java.util.Optional; import org.springframework.ai.document.Document; +import org.springframework.ai.document.DocumentWriter; -public interface VectorStore { +public interface VectorStore extends DocumentWriter { /** * Adds Documents to the vector store. @@ -14,6 +15,10 @@ public interface VectorStore { */ void add(List documents); + default void accept(List documents) { + add(documents); + } + Optional delete(List idList); List similaritySearch(String query); diff --git a/spring-ai-core/src/test/java/org/springframework/ai/loader/JsonLoaderTests.java b/spring-ai-core/src/test/java/org/springframework/ai/reader/JsonLoaderTests.java similarity index 86% rename from spring-ai-core/src/test/java/org/springframework/ai/loader/JsonLoaderTests.java rename to spring-ai-core/src/test/java/org/springframework/ai/reader/JsonLoaderTests.java index 0a816e81d..3d3010246 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/loader/JsonLoaderTests.java +++ b/spring-ai-core/src/test/java/org/springframework/ai/reader/JsonLoaderTests.java @@ -14,11 +14,11 @@ * limitations under the License. */ -package org.springframework.ai.loader; +package org.springframework.ai.reader; import org.junit.jupiter.api.Test; import org.springframework.ai.document.Document; -import org.springframework.ai.loader.impl.JsonLoader; +import org.springframework.ai.reader.JsonReader; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.io.Resource; @@ -36,8 +36,8 @@ public class JsonLoaderTests { @Test void loadJson() { assertThat(resource).isNotNull(); - JsonLoader jsonLoader = new JsonLoader(resource, "description"); - List documents = jsonLoader.load(); + JsonReader jsonLoader = new JsonReader(resource, "description"); + List documents = jsonLoader.get(); assertThat(documents).isNotEmpty(); for (Document document : documents) { assertThat(document.getContent()).isNotEmpty(); diff --git a/spring-ai-core/src/test/java/org/springframework/ai/loader/TextLoaderTests.java b/spring-ai-core/src/test/java/org/springframework/ai/reader/TextLoaderTests.java similarity index 76% rename from spring-ai-core/src/test/java/org/springframework/ai/loader/TextLoaderTests.java rename to spring-ai-core/src/test/java/org/springframework/ai/reader/TextLoaderTests.java index a9043bcbd..fb8366bef 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/loader/TextLoaderTests.java +++ b/spring-ai-core/src/test/java/org/springframework/ai/reader/TextLoaderTests.java @@ -14,14 +14,15 @@ * limitations under the License. */ -package org.springframework.ai.loader; +package org.springframework.ai.reader; import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.ai.document.Document; -import org.springframework.ai.loader.impl.TextLoader; +import org.springframework.ai.reader.TextReader; +import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.core.io.DefaultResourceLoader; import org.springframework.core.io.Resource; @@ -37,17 +38,19 @@ public class TextLoaderTests { @Test void loadText() { assertThat(resource).isNotNull(); - TextLoader textLoader = new TextLoader(resource); + TextReader textLoader = new TextReader(resource); textLoader.getCustomMetadata().put("customKey", "Value"); - List documents = textLoader.load(); + List documents0 = textLoader.get(); + + List documents = new TokenTextSplitter().apply(documents0); assertThat(documents.size()).isEqualTo(54); for (Document document : documents) { assertThat(document.getMetadata().get("customKey")).isEqualTo("Value"); - assertThat(document.getMetadata().get(TextLoader.SOURCE_METADATA)).isEqualTo("text_source.txt"); - assertThat(document.getMetadata().get(TextLoader.CHARSET_METADATA)).isEqualTo("UTF-8"); + assertThat(document.getMetadata().get(TextReader.SOURCE_METADATA)).isEqualTo("text_source.txt"); + assertThat(document.getMetadata().get(TextReader.CHARSET_METADATA)).isEqualTo("UTF-8"); assertThat(document.getContent()).isNotEmpty(); } } diff --git a/spring-ai-core/src/test/java/org/springframework/ai/splitter/TextSplitterTests.java b/spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TextSplitterTests.java similarity index 97% rename from spring-ai-core/src/test/java/org/springframework/ai/splitter/TextSplitterTests.java rename to spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TextSplitterTests.java index ddb58972a..43e724fdc 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/splitter/TextSplitterTests.java +++ b/spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TextSplitterTests.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.splitter; +package org.springframework.ai.transformer.splitter; import java.util.ArrayList; import java.util.List; @@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test; import org.springframework.ai.document.DefaultContentFormatter; import org.springframework.ai.document.Document; +import org.springframework.ai.transformer.splitter.TextSplitter; /** * @author Christian Tzolov diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIT.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIT.java index 23cbac8d0..cc49ad94a 100644 --- a/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIT.java +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIT.java @@ -6,16 +6,17 @@ import org.slf4j.LoggerFactory; import org.springframework.ai.client.AiClient; import org.springframework.ai.client.AiResponse; import org.springframework.ai.document.Document; -import org.springframework.ai.loader.impl.JsonLoader; import org.springframework.ai.openai.embedding.OpenAiEmbeddingClient; import org.springframework.ai.openai.testutils.AbstractIT; import org.springframework.ai.prompt.Prompt; import org.springframework.ai.prompt.SystemPromptTemplate; import org.springframework.ai.prompt.messages.Message; import org.springframework.ai.prompt.messages.UserMessage; -import org.springframework.ai.retriever.impl.VectorStoreRetriever; +import org.springframework.ai.reader.JsonReader; +import org.springframework.ai.retriever.VectorStoreRetriever; +import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.vectorstore.InMemoryVectorStore; import org.springframework.ai.vectorstore.VectorStore; -import org.springframework.ai.vectorstore.impl.InMemoryVectorStore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; @@ -51,19 +52,20 @@ public class AcmeIT extends AbstractIT { assertThat(aiClient).isNotNull(); } - // @Test + @Test void acmeChain() { // Step 1 - load documents - JsonLoader jsonLoader = new JsonLoader(bikesResource, "name", "price", "shortDescription", "description"); - List documents = jsonLoader.load(); + JsonReader jsonLoader = new JsonReader(bikesResource, "name", "price", "shortDescription", "description"); + + var textSplitter = new TokenTextSplitter(); // Step 2 - Create embeddings and save to vector store logger.info("Creating Embeddings..."); VectorStore vectorStore = new InMemoryVectorStore(embeddingClient); - vectorStore.add(documents); + vectorStore.accept(textSplitter.apply(jsonLoader.get())); // Now user query diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/extractor/MetadataExtractorIT.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/transformer/MetadataTransformerIT.java similarity index 84% rename from spring-ai-openai/src/test/java/org/springframework/ai/openai/extractor/MetadataExtractorIT.java rename to spring-ai-openai/src/test/java/org/springframework/ai/openai/transformer/MetadataTransformerIT.java index bd45c43b4..47e94fcfc 100644 --- a/spring-ai-openai/src/test/java/org/springframework/ai/openai/extractor/MetadataExtractorIT.java +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/transformer/MetadataTransformerIT.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.openai.extractor; +package org.springframework.ai.openai.transformer; import java.io.IOException; import java.time.Duration; @@ -27,11 +27,11 @@ import org.junit.jupiter.api.Test; import org.springframework.ai.document.DefaultContentFormatter; import org.springframework.ai.document.Document; -import org.springframework.ai.loader.extractor.ContentFormatEnricher; -import org.springframework.ai.loader.extractor.KeywordExtractor; -import org.springframework.ai.loader.extractor.SummaryExtractor; -import org.springframework.ai.loader.extractor.SummaryExtractor.SummaryType; import org.springframework.ai.openai.client.OpenAiClient; +import org.springframework.ai.transformer.ContentFormatTransformer; +import org.springframework.ai.transformer.KeywordMetadataEnricher; +import org.springframework.ai.transformer.SummaryMetadataEnricher; +import org.springframework.ai.transformer.SummaryMetadataEnricher.SummaryType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.SpringBootTest; @@ -44,16 +44,16 @@ import static org.assertj.core.api.Assertions.assertThat; * @author Christian Tzolov */ @SpringBootTest -public class MetadataExtractorIT { +public class MetadataTransformerIT { @Autowired - KeywordExtractor keywordExtractor; + KeywordMetadataEnricher keywordMetadataEnricher; @Autowired - SummaryExtractor summaryExtractor; + SummaryMetadataEnricher summaryMetadataEnricher; @Autowired - ContentFormatEnricher metadataExtractor; + ContentFormatTransformer contentFormatTransformer; @Autowired DefaultContentFormatter defaultContentFormatter; @@ -75,7 +75,7 @@ public class MetadataExtractorIT { @Test public void testKeywordExtractor() { - var updatedDocuments = keywordExtractor.apply(List.of(document1, document2)); + var updatedDocuments = keywordMetadataEnricher.apply(List.of(document1, document2)); List> keywords = updatedDocuments.stream().map(d -> d.getMetadata()).toList(); @@ -92,7 +92,7 @@ public class MetadataExtractorIT { @Test public void testSummaryExtractor() { - var updatedDocuments = summaryExtractor.apply(List.of(document1, document2)); + var updatedDocuments = summaryMetadataEnricher.apply(List.of(document1, document2)); List> summaries = updatedDocuments.stream().map(d -> d.getMetadata()).toList(); @@ -126,7 +126,7 @@ public class MetadataExtractorIT { assertThat(((DefaultContentFormatter) document2.getContentFormatter()).getExcludedInferenceMetadataKeys()) .doesNotContain("NewInferenceKey"); - List enrichedDocuments = metadataExtractor.apply(List.of(document1, document2)); + List enrichedDocuments = contentFormatTransformer.apply(List.of(document1, document2)); assertThat(enrichedDocuments.size()).isEqualTo(2); var doc1 = enrichedDocuments.get(0); @@ -173,13 +173,14 @@ public class MetadataExtractorIT { } @Bean - public KeywordExtractor keywordExtractor(OpenAiClient aiClient) { - return new KeywordExtractor(aiClient, 5); + public KeywordMetadataEnricher keywordMetadata(OpenAiClient aiClient) { + return new KeywordMetadataEnricher(aiClient, 5); } @Bean - public SummaryExtractor summaryExtractor(OpenAiClient aiClient) { - return new SummaryExtractor(aiClient, List.of(SummaryType.PREVIOUS, SummaryType.CURRENT, SummaryType.NEXT)); + public SummaryMetadataEnricher summaryMetadata(OpenAiClient aiClient) { + return new SummaryMetadataEnricher(aiClient, + List.of(SummaryType.PREVIOUS, SummaryType.CURRENT, SummaryType.NEXT)); } @Bean @@ -191,8 +192,8 @@ public class MetadataExtractorIT { } @Bean - public ContentFormatEnricher metadataExtractor(DefaultContentFormatter defaultContentFormatter) { - return new ContentFormatEnricher(defaultContentFormatter, false); + public ContentFormatTransformer contentFormatTransformer(DefaultContentFormatter defaultContentFormatter) { + return new ContentFormatTransformer(defaultContentFormatter, false); } } diff --git a/spring-ai-openai/src/test/java/org/springframework/ai/openai/vectorstore/SimplePersistentVectorStoreIT.java b/spring-ai-openai/src/test/java/org/springframework/ai/openai/vectorstore/SimplePersistentVectorStoreIT.java index 1e20ed032..630b11334 100644 --- a/spring-ai-openai/src/test/java/org/springframework/ai/openai/vectorstore/SimplePersistentVectorStoreIT.java +++ b/spring-ai-openai/src/test/java/org/springframework/ai/openai/vectorstore/SimplePersistentVectorStoreIT.java @@ -5,9 +5,9 @@ import org.junit.jupiter.api.io.CleanupMode; import org.junit.jupiter.api.io.TempDir; import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingClient; -import org.springframework.ai.loader.impl.JsonLoader; -import org.springframework.ai.loader.impl.JsonMetadataGenerator; -import org.springframework.ai.vectorstore.impl.SimplePersistentVectorStore; +import org.springframework.ai.reader.JsonReader; +import org.springframework.ai.vectorstore.SimplePersistentVectorStore; +import org.springframework.ai.reader.JsonMetadataGenerator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; @@ -31,9 +31,9 @@ public class SimplePersistentVectorStoreIT { @Test void persist(@TempDir(cleanup = CleanupMode.ON_SUCCESS) Path workingDir) { - JsonLoader jsonLoader = new JsonLoader(bikesJsonResource, new ProductMetadataGenerator(), "price", "name", + JsonReader jsonLoader = new JsonReader(bikesJsonResource, new ProductMetadataGenerator(), "price", "name", "shortDescription", "description", "tags"); - List documents = jsonLoader.load(); + List documents = jsonLoader.get(); SimplePersistentVectorStore vectorStore = new SimplePersistentVectorStore(this.embeddingClient); vectorStore.add(documents);