diff --git a/document-readers/markdown-reader/pom.xml b/document-readers/markdown-reader/pom.xml index 54bdb8570..8721693f8 100644 --- a/document-readers/markdown-reader/pom.xml +++ b/document-readers/markdown-reader/pom.xml @@ -40,7 +40,13 @@ org.springframework.ai - spring-ai-core + spring-ai-commons + ${project.parent.version} + + + + org.springframework.ai + spring-ai-document-ingestion ${project.parent.version} diff --git a/document-readers/pdf-reader/pom.xml b/document-readers/pdf-reader/pom.xml index 53ba5b2c7..98733d545 100644 --- a/document-readers/pdf-reader/pom.xml +++ b/document-readers/pdf-reader/pom.xml @@ -40,12 +40,30 @@ + + + org.springframework + spring-context + + org.springframework.ai - spring-ai-core + spring-ai-commons ${project.parent.version} + + org.springframework.ai + spring-ai-document-ingestion + ${project.parent.version} + + + + + org.springframework.boot + spring-boot-starter-logging + + org.apache.pdfbox pdfbox diff --git a/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/config/PdfDocumentReaderConfig.java b/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/config/PdfDocumentReaderConfig.java index d3f804ef5..99d62f3ba 100644 --- a/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/config/PdfDocumentReaderConfig.java +++ b/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/config/PdfDocumentReaderConfig.java @@ -16,7 +16,7 @@ package org.springframework.ai.reader.pdf.config; -import org.springframework.ai.reader.ExtractedTextFormatter; +import org.springframework.ai.document.reader.ExtractedTextFormatter; import org.springframework.ai.reader.pdf.PagePdfDocumentReader; import org.springframework.ai.reader.pdf.ParagraphPdfDocumentReader; import org.springframework.util.Assert; diff --git a/document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/PagePdfDocumentReaderTests.java b/document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/PagePdfDocumentReaderTests.java index 0c9f4d8b9..f311cd5e0 100644 --- a/document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/PagePdfDocumentReaderTests.java +++ b/document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/PagePdfDocumentReaderTests.java @@ -22,7 +22,7 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.springframework.ai.document.Document; -import org.springframework.ai.reader.ExtractedTextFormatter; +import org.springframework.ai.document.reader.ExtractedTextFormatter; import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig; import static org.assertj.core.api.Assertions.assertThat; diff --git a/document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReaderTests.java b/document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReaderTests.java index b514f690e..cb42c36be 100644 --- a/document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReaderTests.java +++ b/document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReaderTests.java @@ -18,7 +18,7 @@ package org.springframework.ai.reader.pdf; import org.junit.jupiter.api.Test; -import org.springframework.ai.reader.ExtractedTextFormatter; +import org.springframework.ai.document.reader.ExtractedTextFormatter; import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig; import static org.assertj.core.api.Assertions.assertThatThrownBy; diff --git a/document-readers/tika-reader/pom.xml b/document-readers/tika-reader/pom.xml index 10e264982..637c029c7 100644 --- a/document-readers/tika-reader/pom.xml +++ b/document-readers/tika-reader/pom.xml @@ -43,7 +43,13 @@ org.springframework.ai - spring-ai-core + spring-ai-commons + ${project.parent.version} + + + + org.springframework.ai + spring-ai-document-ingestion ${project.parent.version} diff --git a/document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java b/document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java index 88d63f7d2..51845d1d8 100644 --- a/document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java +++ b/document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java @@ -29,7 +29,7 @@ import org.xml.sax.ContentHandler; import org.springframework.ai.document.Document; import org.springframework.ai.document.DocumentReader; -import org.springframework.ai.reader.ExtractedTextFormatter; +import org.springframework.ai.document.reader.ExtractedTextFormatter; import org.springframework.core.io.DefaultResourceLoader; import org.springframework.core.io.Resource; import org.springframework.util.StringUtils; diff --git a/document-readers/tika-reader/src/test/java/org/springframework/ai/reader/tika/TikaDocumentReaderTests.java b/document-readers/tika-reader/src/test/java/org/springframework/ai/reader/tika/TikaDocumentReaderTests.java index 5ae1e7a7d..a7c957211 100644 --- a/document-readers/tika-reader/src/test/java/org/springframework/ai/reader/tika/TikaDocumentReaderTests.java +++ b/document-readers/tika-reader/src/test/java/org/springframework/ai/reader/tika/TikaDocumentReaderTests.java @@ -43,7 +43,7 @@ public class TikaDocumentReaderTests { assertThat(doc.getMetadata()).containsKeys(TikaDocumentReader.METADATA_SOURCE); assertThat(doc.getMetadata().get(TikaDocumentReader.METADATA_SOURCE)).isEqualTo(resourceName); - assertThat(doc.getContent()).contains(contentSnipped); + assertThat(doc.getText()).contains(contentSnipped); } } diff --git a/document-transformers/chat-document-transformer/pom.xml b/document-transformers/chat-document-transformer/pom.xml new file mode 100644 index 000000000..b33ee9659 --- /dev/null +++ b/document-transformers/chat-document-transformer/pom.xml @@ -0,0 +1,71 @@ + + + + + 4.0.0 + + org.springframework.ai + spring-ai + 1.0.0-SNAPSHOT + ../../pom.xml + + spring-ai-chat-document-transformer + jar + Spring AI Document Transformer - Chat + Spring AI Document Transformers that use Chat + https://github.com/spring-projects/spring-ai + + + https://github.com/spring-projects/spring-ai + git://github.com/spring-projects/spring-ai.git + git@github.com:spring-projects/spring-ai.git + + + + + + org.springframework.ai + spring-ai-commons + ${project.parent.version} + + + + org.springframework.ai + spring-ai-document-ingestion + ${project.parent.version} + + + + + + org.springframework.ai + spring-ai-core + ${project.parent.version} + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + diff --git a/spring-ai-core/src/main/java/org/springframework/ai/transformer/KeywordMetadataEnricher.java b/document-transformers/chat-document-transformer/src/main/java/org/springframework/ai/document/transformer/chat/KeywordMetadataEnricher.java similarity index 95% rename from spring-ai-core/src/main/java/org/springframework/ai/transformer/KeywordMetadataEnricher.java rename to document-transformers/chat-document-transformer/src/main/java/org/springframework/ai/document/transformer/chat/KeywordMetadataEnricher.java index ff5a4a5d1..67096c6d8 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/transformer/KeywordMetadataEnricher.java +++ b/document-transformers/chat-document-transformer/src/main/java/org/springframework/ai/document/transformer/chat/KeywordMetadataEnricher.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.transformer; +package org.springframework.ai.document.transformer.chat; import java.util.List; import java.util.Map; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/transformer/SummaryMetadataEnricher.java b/document-transformers/chat-document-transformer/src/main/java/org/springframework/ai/document/transformer/chat/SummaryMetadataEnricher.java similarity index 97% rename from spring-ai-core/src/main/java/org/springframework/ai/transformer/SummaryMetadataEnricher.java rename to document-transformers/chat-document-transformer/src/main/java/org/springframework/ai/document/transformer/chat/SummaryMetadataEnricher.java index 4f1fe394d..4a9814c12 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/transformer/SummaryMetadataEnricher.java +++ b/document-transformers/chat-document-transformer/src/main/java/org/springframework/ai/document/transformer/chat/SummaryMetadataEnricher.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.transformer; +package org.springframework.ai.document.transformer.chat; import java.util.ArrayList; import java.util.HashMap; diff --git a/models/spring-ai-openai/pom.xml b/models/spring-ai-openai/pom.xml index 1be48d84d..b200683e2 100644 --- a/models/spring-ai-openai/pom.xml +++ b/models/spring-ai-openai/pom.xml @@ -48,6 +48,13 @@ ${project.parent.version} + + org.springframework.ai + spring-ai-chat-document-transformer + ${project.parent.version} + + + org.springframework.ai spring-ai-retry @@ -82,6 +89,11 @@ + + org.springframework.ai + spring-ai-document-ingestion + ${project.parent.version} + org.springframework.ai spring-ai-vector-store diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIT.java index 2701a2fe8..71102c813 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/acme/AcmeIT.java @@ -35,8 +35,8 @@ import org.springframework.ai.openai.OpenAiChatModel; import org.springframework.ai.openai.OpenAiEmbeddingModel; import org.springframework.ai.openai.OpenAiTestConfiguration; import org.springframework.ai.openai.testutils.AbstractIT; -import org.springframework.ai.reader.JsonReader; -import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.document.reader.JsonReader; +import org.springframework.ai.document.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.SimpleVectorStore; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Autowired; diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/transformer/MetadataTransformerIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/transformer/MetadataTransformerIT.java index 66a8d3d8e..999bec6bf 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/transformer/MetadataTransformerIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/transformer/MetadataTransformerIT.java @@ -28,10 +28,10 @@ import org.springframework.ai.document.DefaultContentFormatter; import org.springframework.ai.document.Document; import org.springframework.ai.openai.OpenAiChatModel; import org.springframework.ai.openai.api.OpenAiApi; -import org.springframework.ai.transformer.ContentFormatTransformer; -import org.springframework.ai.transformer.KeywordMetadataEnricher; -import org.springframework.ai.transformer.SummaryMetadataEnricher; -import org.springframework.ai.transformer.SummaryMetadataEnricher.SummaryType; +import org.springframework.ai.document.transformer.ContentFormatTransformer; +import org.springframework.ai.document.transformer.chat.KeywordMetadataEnricher; +import org.springframework.ai.document.transformer.chat.SummaryMetadataEnricher; +import org.springframework.ai.document.transformer.chat.SummaryMetadataEnricher.SummaryType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.SpringBootTest; diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/vectorstore/SimplePersistentVectorStoreIT.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/vectorstore/SimplePersistentVectorStoreIT.java index 4898e4a68..fe62e6ffa 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/vectorstore/SimplePersistentVectorStoreIT.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/vectorstore/SimplePersistentVectorStoreIT.java @@ -28,8 +28,8 @@ import org.junit.jupiter.api.io.TempDir; import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; -import org.springframework.ai.reader.JsonMetadataGenerator; -import org.springframework.ai.reader.JsonReader; +import org.springframework.ai.document.reader.JsonMetadataGenerator; +import org.springframework.ai.document.reader.JsonReader; import org.springframework.ai.vectorstore.SimpleVectorStore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; diff --git a/pom.xml b/pom.xml index 82c79473e..1dc0d9068 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,8 @@ spring-ai-docs spring-ai-bom + spring-ai-commons + spring-ai-document-ingestion spring-ai-core spring-ai-vector-store spring-ai-test @@ -50,6 +52,7 @@ document-readers/markdown-reader document-readers/pdf-reader document-readers/tika-reader + document-transformers/chat-document-transformer vector-stores/spring-ai-azure-cosmos-db-store vector-stores/spring-ai-azure-store diff --git a/spring-ai-commons/pom.xml b/spring-ai-commons/pom.xml new file mode 100644 index 000000000..09a84bf68 --- /dev/null +++ b/spring-ai-commons/pom.xml @@ -0,0 +1,71 @@ + + + + + 4.0.0 + + org.springframework.ai + spring-ai + 1.0.0-SNAPSHOT + + spring-ai-commons + jar + Spring AI Commons + Spring AI Commons + https://github.com/spring-projects/spring-ai + + + https://github.com/spring-projects/spring-ai + git://github.com/spring-projects/spring-ai.git + git@github.com:spring-projects/spring-ai.git + + + + + + + + + + org.springframework + spring-context + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/ContentFormatter.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/ContentFormatter.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/ContentFormatter.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/ContentFormatter.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/DefaultContentFormatter.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/DefaultContentFormatter.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/DefaultContentFormatter.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/DefaultContentFormatter.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/Document.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/Document.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/Document.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/Document.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentMetadata.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/DocumentMetadata.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/DocumentMetadata.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/DocumentMetadata.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/MetadataMode.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/MetadataMode.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/MetadataMode.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/MetadataMode.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/id/IdGenerator.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/id/IdGenerator.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/id/IdGenerator.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/id/IdGenerator.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/id/JdkSha256HexIdGenerator.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/id/JdkSha256HexIdGenerator.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/id/JdkSha256HexIdGenerator.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/id/JdkSha256HexIdGenerator.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/id/RandomIdGenerator.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/id/RandomIdGenerator.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/id/RandomIdGenerator.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/id/RandomIdGenerator.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/package-info.java b/spring-ai-commons/src/main/java/org/springframework/ai/document/package-info.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/package-info.java rename to spring-ai-commons/src/main/java/org/springframework/ai/document/package-info.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/model/Media.java b/spring-ai-commons/src/main/java/org/springframework/ai/model/Media.java similarity index 99% rename from spring-ai-core/src/main/java/org/springframework/ai/model/Media.java rename to spring-ai-commons/src/main/java/org/springframework/ai/model/Media.java index 5a3ae4bd1..550717fbb 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/model/Media.java +++ b/spring-ai-commons/src/main/java/org/springframework/ai/model/Media.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-ai-core/src/test/java/org/springframework/ai/document/ContentFormatterTests.java b/spring-ai-commons/src/test/java/org/springframework/ai/document/ContentFormatterTests.java similarity index 100% rename from spring-ai-core/src/test/java/org/springframework/ai/document/ContentFormatterTests.java rename to spring-ai-commons/src/test/java/org/springframework/ai/document/ContentFormatterTests.java diff --git a/spring-ai-core/src/test/java/org/springframework/ai/document/DocumentBuilderTests.java b/spring-ai-commons/src/test/java/org/springframework/ai/document/DocumentBuilderTests.java similarity index 100% rename from spring-ai-core/src/test/java/org/springframework/ai/document/DocumentBuilderTests.java rename to spring-ai-commons/src/test/java/org/springframework/ai/document/DocumentBuilderTests.java diff --git a/spring-ai-core/src/test/java/org/springframework/ai/document/DocumentTests.java b/spring-ai-commons/src/test/java/org/springframework/ai/document/DocumentTests.java similarity index 100% rename from spring-ai-core/src/test/java/org/springframework/ai/document/DocumentTests.java rename to spring-ai-commons/src/test/java/org/springframework/ai/document/DocumentTests.java diff --git a/spring-ai-core/src/test/java/org/springframework/ai/document/id/IdGeneratorProviderTest.java b/spring-ai-commons/src/test/java/org/springframework/ai/document/id/IdGeneratorProviderTest.java similarity index 100% rename from spring-ai-core/src/test/java/org/springframework/ai/document/id/IdGeneratorProviderTest.java rename to spring-ai-commons/src/test/java/org/springframework/ai/document/id/IdGeneratorProviderTest.java diff --git a/spring-ai-core/src/test/java/org/springframework/ai/document/id/JdkSha256HexIdGeneratorTest.java b/spring-ai-commons/src/test/java/org/springframework/ai/document/id/JdkSha256HexIdGeneratorTest.java similarity index 100% rename from spring-ai-core/src/test/java/org/springframework/ai/document/id/JdkSha256HexIdGeneratorTest.java rename to spring-ai-commons/src/test/java/org/springframework/ai/document/id/JdkSha256HexIdGeneratorTest.java diff --git a/spring-ai-core/pom.xml b/spring-ai-core/pom.xml index c1bcb0f3b..1a9886e37 100644 --- a/spring-ai-core/pom.xml +++ b/spring-ai-core/pom.xml @@ -41,6 +41,11 @@ + + org.springframework.ai + spring-ai-commons + ${project.parent.version} + com.fasterxml.jackson.module diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/hana.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/hana.adoc index 180c72099..8cbf44c51 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/hana.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/hana.adoc @@ -245,7 +245,7 @@ import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.chat.prompt.SystemPromptTemplate; import org.springframework.ai.document.Document; import org.springframework.ai.reader.pdf.PagePdfDocumentReader; -import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.document.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.HanaCloudVectorStore; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Autowired; diff --git a/spring-ai-document-ingestion/pom.xml b/spring-ai-document-ingestion/pom.xml new file mode 100644 index 000000000..c0c42287e --- /dev/null +++ b/spring-ai-document-ingestion/pom.xml @@ -0,0 +1,84 @@ + + + + + 4.0.0 + + org.springframework.ai + spring-ai + 1.0.0-SNAPSHOT + + spring-ai-document-ingestion + jar + Spring AI Document Ingestion + Spring AI Document Ingestion + https://github.com/spring-projects/spring-ai + + + https://github.com/spring-projects/spring-ai + git://github.com/spring-projects/spring-ai.git + git@github.com:spring-projects/spring-ai.git + + + + + + + + + + org.springframework.ai + spring-ai-commons + ${project.parent.version} + + + + + org.springframework + spring-core + + + + com.knuddels + jtokkit + ${jtokkit.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentReader.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/DocumentReader.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/DocumentReader.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/DocumentReader.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentTransformer.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/DocumentTransformer.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/DocumentTransformer.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/DocumentTransformer.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/document/DocumentWriter.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/DocumentWriter.java similarity index 100% rename from spring-ai-core/src/main/java/org/springframework/ai/document/DocumentWriter.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/DocumentWriter.java diff --git a/spring-ai-core/src/main/java/org/springframework/ai/reader/EmptyJsonMetadataGenerator.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/EmptyJsonMetadataGenerator.java similarity index 95% rename from spring-ai-core/src/main/java/org/springframework/ai/reader/EmptyJsonMetadataGenerator.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/EmptyJsonMetadataGenerator.java index 9ba62979f..dc37a3808 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/reader/EmptyJsonMetadataGenerator.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/EmptyJsonMetadataGenerator.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.reader; +package org.springframework.ai.document.reader; import java.util.Collections; import java.util.Map; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/reader/ExtractedTextFormatter.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/ExtractedTextFormatter.java similarity index 99% rename from spring-ai-core/src/main/java/org/springframework/ai/reader/ExtractedTextFormatter.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/ExtractedTextFormatter.java index 31112672c..09b34d400 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/reader/ExtractedTextFormatter.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/ExtractedTextFormatter.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.reader; +package org.springframework.ai.document.reader; import org.springframework.util.StringUtils; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/reader/JsonMetadataGenerator.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/JsonMetadataGenerator.java similarity index 95% rename from spring-ai-core/src/main/java/org/springframework/ai/reader/JsonMetadataGenerator.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/JsonMetadataGenerator.java index a556e8b65..1f334e98c 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/reader/JsonMetadataGenerator.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/JsonMetadataGenerator.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.reader; +package org.springframework.ai.document.reader; import java.util.Map; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/reader/JsonReader.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/JsonReader.java similarity index 98% rename from spring-ai-core/src/main/java/org/springframework/ai/reader/JsonReader.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/JsonReader.java index 2ea446400..09d746c4e 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/reader/JsonReader.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/JsonReader.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.reader; +package org.springframework.ai.document.reader; import java.io.IOException; import java.util.Collections; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/reader/TextReader.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/TextReader.java similarity index 98% rename from spring-ai-core/src/main/java/org/springframework/ai/reader/TextReader.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/TextReader.java index 9e2a84719..0aa0a933d 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/reader/TextReader.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/reader/TextReader.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.reader; +package org.springframework.ai.document.reader; import java.io.IOException; import java.net.URI; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/transformer/ContentFormatTransformer.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/ContentFormatTransformer.java similarity index 97% rename from spring-ai-core/src/main/java/org/springframework/ai/transformer/ContentFormatTransformer.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/ContentFormatTransformer.java index 32e201402..40069b1dc 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/transformer/ContentFormatTransformer.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/ContentFormatTransformer.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.transformer; +package org.springframework.ai.document.transformer; import java.util.ArrayList; import java.util.List; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TextSplitter.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/splitter/TextSplitter.java similarity index 91% rename from spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TextSplitter.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/splitter/TextSplitter.java index 7d5439c86..509782be2 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TextSplitter.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/splitter/TextSplitter.java @@ -14,15 +14,15 @@ * limitations under the License. */ -package org.springframework.ai.transformer.splitter; +package org.springframework.ai.document.transformer.splitter; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; import org.springframework.ai.document.ContentFormatter; import org.springframework.ai.document.Document; @@ -30,7 +30,7 @@ import org.springframework.ai.document.DocumentTransformer; public abstract class TextSplitter implements DocumentTransformer { - private static final Logger logger = LoggerFactory.getLogger(TextSplitter.class); + // private static final Logger logger = LoggerFactory.getLogger(TextSplitter.class); /** * If true the children documents inherit the content-type of the parent they were @@ -84,7 +84,8 @@ public abstract class TextSplitter implements DocumentTransformer { Map metadata = metadataList.get(i); List chunks = splitText(text); if (chunks.size() > 1) { - logger.info("Splitting up document into " + chunks.size() + " chunks."); + // logger.info("Splitting up document into " + chunks.size() + " + // chunks."); } for (String chunk : chunks) { // only primitive values are in here - diff --git a/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/splitter/TokenTextSplitter.java similarity index 98% rename from spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/splitter/TokenTextSplitter.java index f5c68ff17..1e5374200 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/transformer/splitter/TokenTextSplitter.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.transformer.splitter; +package org.springframework.ai.document.transformer.splitter; import java.util.ArrayList; import java.util.List; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/writer/FileDocumentWriter.java b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/writer/FileDocumentWriter.java similarity index 96% rename from spring-ai-core/src/main/java/org/springframework/ai/writer/FileDocumentWriter.java rename to spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/writer/FileDocumentWriter.java index 8971ef22d..0f6f16aa1 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/writer/FileDocumentWriter.java +++ b/spring-ai-document-ingestion/src/main/java/org/springframework/ai/document/writer/FileDocumentWriter.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.writer; +package org.springframework.ai.document.writer; import java.io.FileWriter; import java.util.List; diff --git a/spring-ai-core/src/test/java/org/springframework/ai/reader/JsonReaderTests.java b/spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/reader/JsonReaderTests.java similarity index 96% rename from spring-ai-core/src/test/java/org/springframework/ai/reader/JsonReaderTests.java rename to spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/reader/JsonReaderTests.java index af7aa19d7..f2c003e67 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/reader/JsonReaderTests.java +++ b/spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/reader/JsonReaderTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.reader; +package org.springframework.ai.document.reader; import java.util.List; diff --git a/spring-ai-core/src/test/java/org/springframework/ai/reader/TextReaderTests.java b/spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/reader/TextReaderTests.java similarity index 95% rename from spring-ai-core/src/test/java/org/springframework/ai/reader/TextReaderTests.java rename to spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/reader/TextReaderTests.java index 835d17f77..c99ac4e72 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/reader/TextReaderTests.java +++ b/spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/reader/TextReaderTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.reader; +package org.springframework.ai.document.reader; import java.nio.charset.StandardCharsets; import java.util.List; @@ -22,7 +22,7 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.ai.document.Document; -import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.document.transformer.splitter.TokenTextSplitter; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.DefaultResourceLoader; import org.springframework.core.io.Resource; diff --git a/spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TextSplitterTests.java b/spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/transformer/splitter/TextSplitterTests.java similarity index 99% rename from spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TextSplitterTests.java rename to spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/transformer/splitter/TextSplitterTests.java index c3172380d..3652d1493 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TextSplitterTests.java +++ b/spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/transformer/splitter/TextSplitterTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.transformer.splitter; +package org.springframework.ai.document.transformer.splitter; import java.util.ArrayList; import java.util.List; diff --git a/spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TokenTextSplitterTest.java b/spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/transformer/splitter/TokenTextSplitterTest.java similarity index 97% rename from spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TokenTextSplitterTest.java rename to spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/transformer/splitter/TokenTextSplitterTest.java index c30225b5c..cd437b0e6 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/transformer/splitter/TokenTextSplitterTest.java +++ b/spring-ai-document-ingestion/src/test/java/org/springframework/ai/document/transformer/splitter/TokenTextSplitterTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.ai.transformer.splitter; +package org.springframework.ai.document.transformer.splitter; import java.util.List; import java.util.Map; diff --git a/spring-ai-vector-store/pom.xml b/spring-ai-vector-store/pom.xml index e0556cec5..1219a57b1 100644 --- a/spring-ai-vector-store/pom.xml +++ b/spring-ai-vector-store/pom.xml @@ -47,6 +47,23 @@ ${project.parent.version} + + org.springframework.ai + spring-ai-document-ingestion + ${project.parent.version} + + + + + + + + + + + + + io.micrometer micrometer-core diff --git a/vector-stores/spring-ai-hanadb-store/src/test/java/org/springframework/ai/vectorstore/CricketWorldCupHanaController.java b/vector-stores/spring-ai-hanadb-store/src/test/java/org/springframework/ai/vectorstore/CricketWorldCupHanaController.java index f3ffbeab9..b76c83b83 100644 --- a/vector-stores/spring-ai-hanadb-store/src/test/java/org/springframework/ai/vectorstore/CricketWorldCupHanaController.java +++ b/vector-stores/spring-ai-hanadb-store/src/test/java/org/springframework/ai/vectorstore/CricketWorldCupHanaController.java @@ -32,7 +32,7 @@ import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.chat.prompt.SystemPromptTemplate; import org.springframework.ai.document.Document; import org.springframework.ai.reader.pdf.PagePdfDocumentReader; -import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.document.transformer.splitter.TokenTextSplitter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; diff --git a/vector-stores/spring-ai-hanadb-store/src/test/java/org/springframework/ai/vectorstore/HanaCloudVectorStoreIT.java b/vector-stores/spring-ai-hanadb-store/src/test/java/org/springframework/ai/vectorstore/HanaCloudVectorStoreIT.java index c55f52cec..aec9c4b61 100644 --- a/vector-stores/spring-ai-hanadb-store/src/test/java/org/springframework/ai/vectorstore/HanaCloudVectorStoreIT.java +++ b/vector-stores/spring-ai-hanadb-store/src/test/java/org/springframework/ai/vectorstore/HanaCloudVectorStoreIT.java @@ -33,7 +33,7 @@ import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.openai.OpenAiEmbeddingModel; import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.ai.reader.pdf.PagePdfDocumentReader; -import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.document.transformer.splitter.TokenTextSplitter; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.runner.ApplicationContextRunner;