Migrage reader/transformer/writer packages from core to commons

Signed-off-by: Soby Chacko <soby.chacko@broadcom.com>

spring-ai-commons/src/main/java/org/springframework/ai/reader/EmptyJsonMetadataGenerator.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader;
+
+import java.util.Collections;
+import java.util.Map;
+
+public class EmptyJsonMetadataGenerator implements JsonMetadataGenerator {
+
+	private static final Map<String, Object> EMPTY_MAP = Collections.emptyMap();
+
+	@Override
+	public Map<String, Object> generate(Map<String, Object> jsonMap) {
+		return EMPTY_MAP;
+	}
+
+}

spring-ai-commons/src/main/java/org/springframework/ai/reader/ExtractedTextFormatter.java

@@ -0,0 +1,308 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader;
+
+import org.springframework.util.StringUtils;
+
+/**
+ * A utility to reformat extracted text content before encapsulating it in a
+ * {@link org.springframework.ai.document.Document}. This formatter provides the following
+ * functionalities:
+ *
+ * <ul>
+ * <li>Left alignment of text</li>
+ * <li>Removal of specified lines from the beginning and end of content</li>
+ * <li>Consolidation of consecutive blank lines</li>
+ * </ul>
+ *
+ * An instance of this formatter can be customized using the {@link Builder} nested class.
+ *
+ * @author Christian Tzolov
+ */
+public final class ExtractedTextFormatter {
+
+	/** Flag indicating if the text should be left-aligned */
+	private final boolean leftAlignment;
+
+	/** Number of top pages to skip before performing delete operations */
+	private final int numberOfTopPagesToSkipBeforeDelete;
+
+	/** Number of top text lines to delete from a page */
+	private final int numberOfTopTextLinesToDelete;
+
+	/** Number of bottom text lines to delete from a page */
+	private final int numberOfBottomTextLinesToDelete;
+
+	/** Line separator */
+	private final String lineSeparator;
+
+	/**
+	 * Private constructor to initialize the formatter from the builder.
+	 * @param builder Builder used to initialize the formatter.
+	 */
+	private ExtractedTextFormatter(Builder builder) {
+		this.leftAlignment = builder.leftAlignment;
+		this.numberOfBottomTextLinesToDelete = builder.numberOfBottomTextLinesToDelete;
+		this.numberOfTopPagesToSkipBeforeDelete = builder.numberOfTopPagesToSkipBeforeDelete;
+		this.numberOfTopTextLinesToDelete = builder.numberOfTopTextLinesToDelete;
+		this.lineSeparator = builder.lineSeparator;
+	}
+
+	/**
+	 * Provides an instance of the builder for this formatter.
+	 * @return an instance of the builder.
+	 */
+	public static Builder builder() {
+		return new Builder();
+	}
+
+	/**
+	 * Provides a default instance of the formatter.
+	 * @return default instance of the formatter.
+	 */
+	public static ExtractedTextFormatter defaults() {
+		return new Builder().build();
+	}
+
+	/**
+	 * Replaces multiple, adjacent blank lines into a single blank line.
+	 * @param pageText text to adjust the blank lines for.
+	 * @return Returns the same text but with blank lines trimmed.
+	 */
+	public static String trimAdjacentBlankLines(String pageText) {
+		return pageText.replaceAll("(?m)(^ *\n)", "\n").replaceAll("(?m)^$([\r\n]+?)(^$[\r\n]+?^)+", "$1");
+	}
+
+	/**
+	 * @param pageText text to align.
+	 * @return Returns the same text but aligned to the left side.
+	 */
+	public static String alignToLeft(String pageText) {
+		return pageText.replaceAll("(?m)(^ *| +(?= |$))", "").replaceAll("(?m)^$(	?)(^$[\r\n]+?^)+", "$1");
+	}
+
+	/**
+	 * Removes the specified number of lines from the bottom part of the text.
+	 * @param pageText Text to remove lines from.
+	 * @param numberOfLines Number of lines to remove.
+	 * @param lineSeparator The line separator to use when identifying lines in the text.
+	 * @return Returns the text striped from last lines.
+	 */
+	public static String deleteBottomTextLines(String pageText, int numberOfLines, String lineSeparator) {
+		if (!StringUtils.hasText(pageText)) {
+			return pageText;
+		}
+
+		int lineCount = 0;
+		int truncateIndex = pageText.length();
+		int nextTruncateIndex = truncateIndex;
+		while (lineCount < numberOfLines && nextTruncateIndex >= 0) {
+			nextTruncateIndex = pageText.lastIndexOf(lineSeparator, truncateIndex - 1);
+			truncateIndex = nextTruncateIndex < 0 ? truncateIndex : nextTruncateIndex;
+			lineCount++;
+		}
+		return pageText.substring(0, truncateIndex);
+	}
+
+	/**
+	 * Removes a specified number of lines from the top part of the given text.
+	 *
+	 * <p>
+	 * This method takes a text and trims it by removing a certain number of lines from
+	 * the top. If the provided text is null or contains only whitespace, it will be
+	 * returned as is. If the number of lines to remove exceeds the actual number of lines
+	 * in the text, the result will be an empty string.
+	 * </p>
+	 *
+	 * <p>
+	 * The method identifies lines based on the system's line separator, making it
+	 * compatible with different platforms.
+	 * </p>
+	 * @param pageText The text from which the top lines need to be removed. If this is
+	 * null, empty, or consists only of whitespace, it will be returned unchanged.
+	 * @param numberOfLines The number of lines to remove from the top of the text. If
+	 * this exceeds the actual number of lines in the text, an empty string will be
+	 * returned.
+	 * @param lineSeparator The line separator to use when identifying lines in the text.
+	 * @return The text with the specified number of lines removed from the top.
+	 */
+	public static String deleteTopTextLines(String pageText, int numberOfLines, String lineSeparator) {
+		if (!StringUtils.hasText(pageText)) {
+			return pageText;
+		}
+		int lineCount = 0;
+
+		int truncateIndex = 0;
+		int nextTruncateIndex = truncateIndex;
+		while (lineCount < numberOfLines && nextTruncateIndex >= 0) {
+			nextTruncateIndex = pageText.indexOf(lineSeparator, truncateIndex + 1);
+			truncateIndex = nextTruncateIndex < 0 ? truncateIndex : nextTruncateIndex;
+			lineCount++;
+		}
+		return pageText.substring(truncateIndex);
+	}
+
+	/**
+	 * Formats the provided text according to the formatter's configuration.
+	 * @param pageText Text to be formatted.
+	 * @return Formatted text.
+	 */
+	public String format(String pageText) {
+		return this.format(pageText, 0);
+	}
+
+	/**
+	 * Formats the provided text based on the formatter's configuration, considering the
+	 * page number.
+	 * @param pageText Text to be formatted.
+	 * @param pageNumber Page number of the provided text.
+	 * @return Formatted text.
+	 */
+	public String format(String pageText, int pageNumber) {
+
+		var text = trimAdjacentBlankLines(pageText);
+
+		if (pageNumber >= this.numberOfTopPagesToSkipBeforeDelete) {
+			text = deleteTopTextLines(text, this.numberOfTopTextLinesToDelete, this.lineSeparator);
+			text = deleteBottomTextLines(text, this.numberOfBottomTextLinesToDelete, this.lineSeparator);
+		}
+
+		if (this.leftAlignment) {
+			text = alignToLeft(text);
+		}
+
+		return text;
+	}
+
+	/**
+	 * The {@code Builder} class is a nested static class of
+	 * {@link ExtractedTextFormatter} designed to facilitate the creation and
+	 * customization of instances of {@link ExtractedTextFormatter}.
+	 *
+	 * <p>
+	 * It allows for a step-by-step, fluent construction of the
+	 * {@link ExtractedTextFormatter}, by providing methods to set specific configurations
+	 * such as left alignment of text, the number of top lines or bottom lines to delete,
+	 * and the number of top pages to skip before deletion. Each configuration method in
+	 * the builder returns the builder instance itself, enabling method chaining.
+	 * </p>
+	 *
+	 *
+	 * By default, the builder sets:
+	 * <ul>
+	 * <li>Left alignment to {@code false}</li>
+	 * <li>Number of top pages to skip before deletion to 0</li>
+	 * <li>Number of top text lines to delete to 0</li>
+	 * <li>Number of bottom text lines to delete to 0</li>
+	 * </ul>
+	 *
+	 *
+	 * <p>
+	 * After configuring the builder, calling the {@link #build()} method will return a
+	 * new instance of {@link ExtractedTextFormatter} with the specified configurations.
+	 * </p>
+	 *
+	 * @see ExtractedTextFormatter
+	 */
+	public static class Builder {
+
+		private boolean leftAlignment = false;
+
+		private int numberOfTopPagesToSkipBeforeDelete = 0;
+
+		private int numberOfTopTextLinesToDelete = 0;
+
+		private int numberOfBottomTextLinesToDelete = 0;
+
+		private String lineSeparator = System.lineSeparator();
+
+		/**
+		 * Align the document text to the left. Defaults to false.
+		 * @param leftAlignment Flag to align the text to the left.
+		 * @return this builder
+		 */
+		public Builder withLeftAlignment(boolean leftAlignment) {
+			this.leftAlignment = leftAlignment;
+			return this;
+		}
+
+		/**
+		 * Withdraw the top N pages from the text top/bottom line deletion. Defaults to 0.
+		 * @param numberOfTopPagesToSkipBeforeDelete Number of pages to skip from
+		 * top/bottom line deletion policy.
+		 * @return this builder
+		 */
+		public Builder withNumberOfTopPagesToSkipBeforeDelete(int numberOfTopPagesToSkipBeforeDelete) {
+			this.numberOfTopPagesToSkipBeforeDelete = numberOfTopPagesToSkipBeforeDelete;
+			return this;
+		}
+
+		/**
+		 * Remove the top N lines from the page text. Defaults to 0.
+		 * @param numberOfTopTextLinesToDelete Number of top text lines to delete.
+		 * @return this builder
+		 */
+		public Builder withNumberOfTopTextLinesToDelete(int numberOfTopTextLinesToDelete) {
+			this.numberOfTopTextLinesToDelete = numberOfTopTextLinesToDelete;
+			return this;
+		}
+
+		/**
+		 * Remove the bottom N lines from the page text. Defaults to 0.
+		 * @param numberOfBottomTextLinesToDelete Number of bottom text lines to delete.
+		 * @return this builder
+		 */
+		public Builder withNumberOfBottomTextLinesToDelete(int numberOfBottomTextLinesToDelete) {
+			this.numberOfBottomTextLinesToDelete = numberOfBottomTextLinesToDelete;
+			return this;
+		}
+
+		/**
+		 * Set the line separator to use when formatting the text. Defaults to the system
+		 * line separator.
+		 * @param lineSeparator The line separator to use.
+		 * @return this builder
+		 */
+		public Builder overrideLineSeparator(String lineSeparator) {
+			this.lineSeparator = lineSeparator;
+			return this;
+		}
+
+		/**
+		 * Constructs and returns an instance of {@link ExtractedTextFormatter} using the
+		 * configurations set on this builder.
+		 *
+		 * <p>
+		 * This method uses the values set on the builder to initialize the configuration
+		 * for the {@link ExtractedTextFormatter} instance. If no values are explicitly
+		 * set on the builder, the defaults specified in the builder are used.
+		 * </p>
+		 *
+		 * <p>
+		 * It's recommended to use this method only once per builder instance to ensure
+		 * that each {@link ExtractedTextFormatter} object is configured as intended.
+		 * </p>
+		 * @return a new instance of {@link ExtractedTextFormatter} configured with the
+		 * values set on this builder.
+		 */
+		public ExtractedTextFormatter build() {
+			return new ExtractedTextFormatter(this);
+		}
+
+	}
+
+}

spring-ai-commons/src/main/java/org/springframework/ai/reader/JsonMetadataGenerator.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader;
+
+import java.util.Map;
+
+@FunctionalInterface
+public interface JsonMetadataGenerator {
+
+	/**
+	 * The input is the JSON document represented as a map, the output are the fields
+	 * extracted from the input map that will be used as metadata.
+	 * @param jsonMap json document map
+	 * @return json metadata map
+	 */
+	Map<String, Object> generate(Map<String, Object> jsonMap);
+
+}

spring-ai-commons/src/main/java/org/springframework/ai/reader/JsonReader.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.StreamSupport;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentReader;
+import org.springframework.core.io.Resource;
+
+/**
+ * A class that reads JSON documents and converts them into a list of {@link Document}
+ * objects.
+ *
+ * @author Mark Pollack
+ * @author Christian Tzolov
+ * @author rivkode rivkode
+ * @since 1.0.0
+ */
+public class JsonReader implements DocumentReader {
+
+	private final Resource resource;
+
+	private final JsonMetadataGenerator jsonMetadataGenerator;
+
+	private final ObjectMapper objectMapper = new ObjectMapper();
+
+	/**
+	 * The key from the JSON that we will use as the text to parse into the Document text
+	 */
+	private final List<String> jsonKeysToUse;
+
+	public JsonReader(Resource resource) {
+		this(resource, new String[0]);
+	}
+
+	public JsonReader(Resource resource, String... jsonKeysToUse) {
+		this(resource, new EmptyJsonMetadataGenerator(), jsonKeysToUse);
+	}
+
+	public JsonReader(Resource resource, JsonMetadataGenerator jsonMetadataGenerator, String... jsonKeysToUse) {
+		Objects.requireNonNull(jsonKeysToUse, "keys must not be null");
+		Objects.requireNonNull(jsonMetadataGenerator, "jsonMetadataGenerator must not be null");
+		Objects.requireNonNull(resource, "The Spring Resource must not be null");
+		this.resource = resource;
+		this.jsonMetadataGenerator = jsonMetadataGenerator;
+		this.jsonKeysToUse = List.of(jsonKeysToUse);
+	}
+
+	@Override
+	public List<Document> get() {
+		try {
+			JsonNode rootNode = this.objectMapper.readTree(this.resource.getInputStream());
+
+			if (rootNode.isArray()) {
+				return StreamSupport.stream(rootNode.spliterator(), true)
+					.map(jsonNode -> parseJsonNode(jsonNode, this.objectMapper))
+					.toList();
+			}
+			else {
+				return Collections.singletonList(parseJsonNode(rootNode, this.objectMapper));
+			}
+		}
+		catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	private Document parseJsonNode(JsonNode jsonNode, ObjectMapper objectMapper) {
+		Map<String, Object> item = objectMapper.convertValue(jsonNode, new TypeReference<Map<String, Object>>() {
+
+		});
+		var sb = new StringBuilder();
+
+		this.jsonKeysToUse.stream()
+			.filter(item::containsKey)
+			.forEach(key -> sb.append(key).append(": ").append(item.get(key)).append(System.lineSeparator()));
+
+		Map<String, Object> metadata = this.jsonMetadataGenerator.generate(item);
+		String content = sb.isEmpty() ? item.toString() : sb.toString();
+		return new Document(content, metadata);
+	}
+
+	protected List<Document> get(JsonNode rootNode) {
+		if (rootNode.isArray()) {
+			return StreamSupport.stream(rootNode.spliterator(), true)
+				.map(jsonNode -> parseJsonNode(jsonNode, this.objectMapper))
+				.toList();
+		}
+		else {
+			return Collections.singletonList(parseJsonNode(rootNode, this.objectMapper));
+		}
+	}
+
+	/**
+	 * Retrieves documents from the JSON resource using a JSON Pointer.
+	 * @param pointer A JSON Pointer string (RFC 6901) to locate the desired element
+	 * @return A list of Documents parsed from the located JSON element
+	 * @throws RuntimeException if the JSON cannot be parsed or the pointer is invalid
+	 */
+	public List<Document> get(String pointer) {
+		try {
+			JsonNode rootNode = this.objectMapper.readTree(this.resource.getInputStream());
+			JsonNode targetNode = rootNode.at(pointer);
+
+			if (targetNode.isMissingNode()) {
+				throw new IllegalArgumentException("Invalid JSON Pointer: " + pointer);
+			}
+
+			return get(targetNode);
+		}
+		catch (IOException e) {
+			throw new RuntimeException("Error reading JSON resource", e);
+		}
+	}
+
+}

spring-ai-commons/src/main/java/org/springframework/ai/reader/TextReader.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentReader;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
+import org.springframework.util.StreamUtils;
+
+/**
+ * A {@link DocumentReader} that reads text from a {@link Resource}.
+ *
+ * @author Craig Walls
+ * @author Christian Tzolov
+ */
+public class TextReader implements DocumentReader {
+
+	public static final String CHARSET_METADATA = "charset";
+
+	public static final String SOURCE_METADATA = "source";
+
+	/**
+	 * Input resource to load the text from.
+	 */
+	private final Resource resource;
+
+	private final Map<String, Object> customMetadata = new HashMap<>();
+
+	/**
+	 * Character set to be used when loading data from the
+	 */
+	private Charset charset = StandardCharsets.UTF_8;
+
+	public TextReader(String resourceUrl) {
+		this(new DefaultResourceLoader().getResource(resourceUrl));
+	}
+
+	public TextReader(Resource resource) {
+		Objects.requireNonNull(resource, "The Spring Resource must not be null");
+		this.resource = resource;
+	}
+
+	public Charset getCharset() {
+		return this.charset;
+	}
+
+	public void setCharset(Charset charset) {
+		Objects.requireNonNull(charset, "The charset must not be null");
+		this.charset = charset;
+	}
+
+	/**
+	 * Metadata associated with all documents created by the loader.
+	 * @return Metadata to be assigned to the output Documents.
+	 */
+	public Map<String, Object> getCustomMetadata() {
+		return this.customMetadata;
+	}
+
+	@Override
+	public List<Document> get() {
+		try {
+
+			String document = StreamUtils.copyToString(this.resource.getInputStream(), this.charset);
+
+			// Inject source information as a metadata.
+			this.customMetadata.put(CHARSET_METADATA, this.charset.name());
+			this.customMetadata.put(SOURCE_METADATA, this.resource.getFilename());
+			this.customMetadata.put(SOURCE_METADATA, getResourceIdentifier(this.resource));
+
+			return List.of(new Document(document, this.customMetadata));
+
+		}
+		catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	protected String getResourceIdentifier(Resource resource) {
+		// Try to get the filename first
+		String filename = resource.getFilename();
+		if (filename != null && !filename.isEmpty()) {
+			return filename;
+		}
+
+		// Try to get the URI
+		try {
+			URI uri = resource.getURI();
+			if (uri != null) {
+				return uri.toString();
+			}
+		}
+		catch (IOException ignored) {
+			// If getURI() throws an exception, we'll try the next method
+		}
+
+		// Try to get the URL
+		try {
+			URL url = resource.getURL();
+			if (url != null) {
+				return url.toString();
+			}
+		}
+		catch (IOException ignored) {
+			// If getURL() throws an exception, we'll fall back to getDescription()
+		}
+
+		// If all else fails, use the description
+		return resource.getDescription();
+	}
+
+}

spring-ai-commons/src/main/java/org/springframework/ai/transformer/ContentFormatTransformer.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.transformer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.ai.document.ContentFormatter;
+import org.springframework.ai.document.DefaultContentFormatter;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentTransformer;
+
+/**
+ * ContentFormatTransformer processes a list of documents by applying a content formatter
+ * to each document.
+ *
+ * @author Christian Tzolov
+ * @since 1.0.0
+ */
+public class ContentFormatTransformer implements DocumentTransformer {
+
+	/**
+	 * Disable the content-formatter template rewrite.
+	 */
+	private final boolean disableTemplateRewrite;
+
+	private final ContentFormatter contentFormatter;
+
+	/**
+	 * Creates a ContentFormatTransformer object with the given ContentFormatter.
+	 * @param contentFormatter the ContentFormatter to be used for transforming the
+	 * documents
+	 */
+	public ContentFormatTransformer(ContentFormatter contentFormatter) {
+		this(contentFormatter, false);
+	}
+
+	/**
+	 * The ContentFormatTransformer class is responsible for processing a list of
+	 * documents by applying a content formatter to each document.
+	 * @param contentFormatter The ContentFormatter to be used for transforming the
+	 * documents
+	 * @param disableTemplateRewrite Flag indicating whether to disable the
+	 * content-formatter template rewrite
+	 */
+	public ContentFormatTransformer(ContentFormatter contentFormatter, boolean disableTemplateRewrite) {
+		this.contentFormatter = contentFormatter;
+		this.disableTemplateRewrite = disableTemplateRewrite;
+	}
+
+	/**
+	 * Post process documents chunked from loader. Allows extractors to be chained.
+	 * @param documents to post process.
+	 * @return processed documents
+	 */
+	public List<Document> apply(List<Document> documents) {
+		if (this.contentFormatter != null) {
+			documents.forEach(this::processDocument);
+		}
+
+		return documents;
+	}
+
+	private void processDocument(Document document) {
+		if (document.getContentFormatter() instanceof DefaultContentFormatter docFormatter
+				&& this.contentFormatter instanceof DefaultContentFormatter toUpdateFormatter) {
+			updateFormatter(document, docFormatter, toUpdateFormatter);
+
+		}
+		else {
+			overrideFormatter(document);
+		}
+	}
+
+	private void updateFormatter(Document document, DefaultContentFormatter docFormatter,
+			DefaultContentFormatter toUpdateFormatter) {
+		List<String> updatedEmbedExcludeKeys = new ArrayList<>(docFormatter.getExcludedEmbedMetadataKeys());
+		updatedEmbedExcludeKeys.addAll(toUpdateFormatter.getExcludedEmbedMetadataKeys());
+
+		List<String> updatedInterfaceExcludeKeys = new ArrayList<>(docFormatter.getExcludedInferenceMetadataKeys());
+		updatedInterfaceExcludeKeys.addAll(toUpdateFormatter.getExcludedInferenceMetadataKeys());
+
+		DefaultContentFormatter.Builder builder = DefaultContentFormatter.builder()
+			.withExcludedEmbedMetadataKeys(updatedEmbedExcludeKeys)
+			.withExcludedInferenceMetadataKeys(updatedInterfaceExcludeKeys)
+			.withMetadataTemplate(docFormatter.getMetadataTemplate())
+			.withMetadataSeparator(docFormatter.getMetadataSeparator());
+
+		if (!this.disableTemplateRewrite) {
+			builder.withTextTemplate(docFormatter.getTextTemplate());
+		}
+
+		document.setContentFormatter(builder.build());
+	}
+
+	private void overrideFormatter(Document document) {
+		document.setContentFormatter(this.contentFormatter);
+	}
+
+}

spring-ai-commons/src/main/java/org/springframework/ai/transformer/splitter/TextSplitter.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.transformer.splitter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.springframework.ai.document.ContentFormatter;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentTransformer;
+
+public abstract class TextSplitter implements DocumentTransformer {
+
+	private static final Logger logger = LoggerFactory.getLogger(TextSplitter.class);
+
+	/**
+	 * If true the children documents inherit the content-type of the parent they were
+	 * split from.
+	 */
+	private boolean copyContentFormatter = true;
+
+	@Override
+	public List<Document> apply(List<Document> documents) {
+		return doSplitDocuments(documents);
+	}
+
+	public List<Document> split(List<Document> documents) {
+		return this.apply(documents);
+	}
+
+	public List<Document> split(Document document) {
+		return this.apply(List.of(document));
+	}
+
+	public boolean isCopyContentFormatter() {
+		return this.copyContentFormatter;
+	}
+
+	public void setCopyContentFormatter(boolean copyContentFormatter) {
+		this.copyContentFormatter = copyContentFormatter;
+	}
+
+	private List<Document> doSplitDocuments(List<Document> documents) {
+		List<String> texts = new ArrayList<>();
+		List<Map<String, Object>> metadataList = new ArrayList<>();
+		List<ContentFormatter> formatters = new ArrayList<>();
+
+		for (Document doc : documents) {
+			texts.add(doc.getText());
+			metadataList.add(doc.getMetadata());
+			formatters.add(doc.getContentFormatter());
+		}
+
+		return createDocuments(texts, formatters, metadataList);
+	}
+
+	private List<Document> createDocuments(List<String> texts, List<ContentFormatter> formatters,
+			List<Map<String, Object>> metadataList) {
+
+		// Process the data in a column oriented way and recreate the Document
+		List<Document> documents = new ArrayList<>();
+
+		for (int i = 0; i < texts.size(); i++) {
+			String text = texts.get(i);
+			Map<String, Object> metadata = metadataList.get(i);
+			List<String> chunks = splitText(text);
+			if (chunks.size() > 1) {
+				logger.info("Splitting up document into " + chunks.size() + " chunks.");
+			}
+			for (String chunk : chunks) {
+				// only primitive values are in here -
+				Map<String, Object> metadataCopy = metadata.entrySet()
+					.stream()
+					.filter(e -> e.getKey() != null && e.getValue() != null)
+					.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+				Document newDoc = new Document(chunk, metadataCopy);
+
+				if (this.copyContentFormatter) {
+					// Transfer the content-formatter of the parent to the chunked
+					// documents it was slit into.
+					newDoc.setContentFormatter(formatters.get(i));
+				}
+
+				// TODO copy over other properties.
+				documents.add(newDoc);
+			}
+		}
+		return documents;
+	}
+
+	protected abstract List<String> splitText(String text);
+
+}

spring-ai-commons/src/main/java/org/springframework/ai/transformer/splitter/TokenTextSplitter.java

@@ -0,0 +1,202 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.transformer.splitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.knuddels.jtokkit.Encodings;
+import com.knuddels.jtokkit.api.Encoding;
+import com.knuddels.jtokkit.api.EncodingRegistry;
+import com.knuddels.jtokkit.api.EncodingType;
+import com.knuddels.jtokkit.api.IntArrayList;
+
+import org.springframework.util.Assert;
+
+/**
+ * A {@link TextSplitter} that splits text into chunks of a target size in tokens.
+ *
+ * @author Raphael Yu
+ * @author Christian Tzolov
+ * @author Ricken Bazolo
+ */
+public class TokenTextSplitter extends TextSplitter {
+
+	private static final int DEFAULT_CHUNK_SIZE = 800;
+
+	private static final int MIN_CHUNK_SIZE_CHARS = 350;
+
+	private static final int MIN_CHUNK_LENGTH_TO_EMBED = 5;
+
+	private static final int MAX_NUM_CHUNKS = 10000;
+
+	private static final boolean KEEP_SEPARATOR = true;
+
+	private final EncodingRegistry registry = Encodings.newLazyEncodingRegistry();
+
+	private final Encoding encoding = this.registry.getEncoding(EncodingType.CL100K_BASE);
+
+	// The target size of each text chunk in tokens
+	private final int chunkSize;
+
+	// The minimum size of each text chunk in characters
+	private final int minChunkSizeChars;
+
+	// Discard chunks shorter than this
+	private final int minChunkLengthToEmbed;
+
+	// The maximum number of chunks to generate from a text
+	private final int maxNumChunks;
+
+	private final boolean keepSeparator;
+
+	public TokenTextSplitter() {
+		this(DEFAULT_CHUNK_SIZE, MIN_CHUNK_SIZE_CHARS, MIN_CHUNK_LENGTH_TO_EMBED, MAX_NUM_CHUNKS, KEEP_SEPARATOR);
+	}
+
+	public TokenTextSplitter(boolean keepSeparator) {
+		this(DEFAULT_CHUNK_SIZE, MIN_CHUNK_SIZE_CHARS, MIN_CHUNK_LENGTH_TO_EMBED, MAX_NUM_CHUNKS, keepSeparator);
+	}
+
+	public TokenTextSplitter(int chunkSize, int minChunkSizeChars, int minChunkLengthToEmbed, int maxNumChunks,
+			boolean keepSeparator) {
+		this.chunkSize = chunkSize;
+		this.minChunkSizeChars = minChunkSizeChars;
+		this.minChunkLengthToEmbed = minChunkLengthToEmbed;
+		this.maxNumChunks = maxNumChunks;
+		this.keepSeparator = keepSeparator;
+	}
+
+	public static Builder builder() {
+		return new Builder();
+	}
+
+	@Override
+	protected List<String> splitText(String text) {
+		return doSplit(text, this.chunkSize);
+	}
+
+	protected List<String> doSplit(String text, int chunkSize) {
+		if (text == null || text.trim().isEmpty()) {
+			return new ArrayList<>();
+		}
+
+		List<Integer> tokens = getEncodedTokens(text);
+		List<String> chunks = new ArrayList<>();
+		int num_chunks = 0;
+		while (!tokens.isEmpty() && num_chunks < this.maxNumChunks) {
+			List<Integer> chunk = tokens.subList(0, Math.min(chunkSize, tokens.size()));
+			String chunkText = decodeTokens(chunk);
+
+			// Skip the chunk if it is empty or whitespace
+			if (chunkText.trim().isEmpty()) {
+				tokens = tokens.subList(chunk.size(), tokens.size());
+				continue;
+			}
+
+			// Find the last period or punctuation mark in the chunk
+			int lastPunctuation = Math.max(chunkText.lastIndexOf('.'), Math.max(chunkText.lastIndexOf('?'),
+					Math.max(chunkText.lastIndexOf('!'), chunkText.lastIndexOf('\n'))));
+
+			if (lastPunctuation != -1 && lastPunctuation > this.minChunkSizeChars) {
+				// Truncate the chunk text at the punctuation mark
+				chunkText = chunkText.substring(0, lastPunctuation + 1);
+			}
+
+			String chunkTextToAppend = (this.keepSeparator) ? chunkText.trim()
+					: chunkText.replace(System.lineSeparator(), " ").trim();
+			if (chunkTextToAppend.length() > this.minChunkLengthToEmbed) {
+				chunks.add(chunkTextToAppend);
+			}
+
+			// Remove the tokens corresponding to the chunk text from the remaining tokens
+			tokens = tokens.subList(getEncodedTokens(chunkText).size(), tokens.size());
+
+			num_chunks++;
+		}
+
+		// Handle the remaining tokens
+		if (!tokens.isEmpty()) {
+			String remaining_text = decodeTokens(tokens).replace(System.lineSeparator(), " ").trim();
+			if (remaining_text.length() > this.minChunkLengthToEmbed) {
+				chunks.add(remaining_text);
+			}
+		}
+
+		return chunks;
+	}
+
+	private List<Integer> getEncodedTokens(String text) {
+		Assert.notNull(text, "Text must not be null");
+		return this.encoding.encode(text).boxed();
+	}
+
+	private String decodeTokens(List<Integer> tokens) {
+		Assert.notNull(tokens, "Tokens must not be null");
+		var tokensIntArray = new IntArrayList(tokens.size());
+		tokens.forEach(tokensIntArray::add);
+		return this.encoding.decode(tokensIntArray);
+	}
+
+	public static final class Builder {
+
+		private int chunkSize;
+
+		private int minChunkSizeChars;
+
+		private int minChunkLengthToEmbed;
+
+		private int maxNumChunks;
+
+		private boolean keepSeparator;
+
+		private Builder() {
+		}
+
+		public Builder withChunkSize(int chunkSize) {
+			this.chunkSize = chunkSize;
+			return this;
+		}
+
+		public Builder withMinChunkSizeChars(int minChunkSizeChars) {
+			this.minChunkSizeChars = minChunkSizeChars;
+			return this;
+		}
+
+		public Builder withMinChunkLengthToEmbed(int minChunkLengthToEmbed) {
+			this.minChunkLengthToEmbed = minChunkLengthToEmbed;
+			return this;
+		}
+
+		public Builder withMaxNumChunks(int maxNumChunks) {
+			this.maxNumChunks = maxNumChunks;
+			return this;
+		}
+
+		public Builder withKeepSeparator(boolean keepSeparator) {
+			this.keepSeparator = keepSeparator;
+			return this;
+		}
+
+		public TokenTextSplitter build() {
+			return new TokenTextSplitter(this.chunkSize, this.minChunkSizeChars, this.minChunkLengthToEmbed,
+					this.maxNumChunks, this.keepSeparator);
+		}
+
+	}
+
+}

spring-ai-commons/src/main/java/org/springframework/ai/writer/FileDocumentWriter.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.writer;
+
+import java.io.FileWriter;
+import java.util.List;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentWriter;
+import org.springframework.ai.document.MetadataMode;
+import org.springframework.util.Assert;
+
+/**
+ * Writes the content of a list of {@link Document}s into a file.
+ *
+ * @author Christian Tzolov
+ */
+public class FileDocumentWriter implements DocumentWriter {
+
+	public static final String METADATA_START_PAGE_NUMBER = "page_number";
+
+	public static final String METADATA_END_PAGE_NUMBER = "end_page_number";
+
+	private final String fileName;
+
+	private final boolean withDocumentMarkers;
+
+	private final MetadataMode metadataMode;
+
+	private final boolean append;
+
+	public FileDocumentWriter(String fileName) {
+		this(fileName, false, MetadataMode.NONE, false);
+	}
+
+	public FileDocumentWriter(String fileName, boolean withDocumentMarkers) {
+		this(fileName, withDocumentMarkers, MetadataMode.NONE, false);
+	}
+
+	/**
+	 * Writes the content of a list of {@link Document}s into a file.
+	 * @param fileName The name of the file to write the documents to.
+	 * @param withDocumentMarkers Whether to include document markers in the output.
+	 * @param metadataMode Document content formatter mode. Specifies what document
+	 * content to be written to the file.
+	 * @param append if {@code true}, then data will be written to the end of the file
+	 * rather than the beginning.
+	 */
+	public FileDocumentWriter(String fileName, boolean withDocumentMarkers, MetadataMode metadataMode, boolean append) {
+		Assert.hasText(fileName, "File name must have a text.");
+		Assert.notNull(metadataMode, "MetadataMode must not be null.");
+
+		this.fileName = fileName;
+		this.withDocumentMarkers = withDocumentMarkers;
+		this.metadataMode = metadataMode;
+		this.append = append;
+	}
+
+	@Override
+	public void accept(List<Document> docs) {
+
+		try (var writer = new FileWriter(this.fileName, this.append)) {
+
+			int index = 0;
+			for (Document doc : docs) {
+				if (this.withDocumentMarkers) {
+					writer.write(String.format("%n### Doc: %s, pages:[%s,%s]\n", index,
+							doc.getMetadata().get(METADATA_START_PAGE_NUMBER),
+							doc.getMetadata().get(METADATA_END_PAGE_NUMBER)));
+				}
+				writer.write(doc.getFormattedContent(this.metadataMode));
+				index++;
+			}
+		}
+		catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+}

spring-ai-commons/src/test/java/org/springframework/ai/TestConfiguration.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai;
+
+import org.springframework.boot.SpringBootConfiguration;
+
+@SpringBootConfiguration
+public class TestConfiguration {
+
+}

spring-ai-commons/src/test/java/org/springframework/ai/reader/JsonReaderTests.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.document.Document;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.core.io.Resource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@SpringBootTest
+public class JsonReaderTests {
+
+	@Value("classpath:person.json")
+	private Resource ObjectResource;
+
+	@Value("classpath:bikes.json")
+	private Resource arrayResource;
+
+	@Value("classpath:events.json")
+	private Resource eventsResource;
+
+	@Test
+	void loadJsonArray() {
+		assertThat(this.arrayResource).isNotNull();
+		JsonReader jsonReader = new JsonReader(this.arrayResource, "description");
+		List<Document> documents = jsonReader.get();
+		assertThat(documents).isNotEmpty();
+		for (Document document : documents) {
+			assertThat(document.getText()).isNotEmpty();
+		}
+	}
+
+	@Test
+	void loadJsonObject() {
+		assertThat(this.ObjectResource).isNotNull();
+		JsonReader jsonReader = new JsonReader(this.ObjectResource, "description");
+		List<Document> documents = jsonReader.get();
+		assertThat(documents).isNotEmpty();
+		for (Document document : documents) {
+			assertThat(document.getText()).isNotEmpty();
+		}
+	}
+
+	@Test
+	void loadJsonArrayFromPointer() {
+		assertThat(this.arrayResource).isNotNull();
+		JsonReader jsonReader = new JsonReader(this.eventsResource, "description");
+		List<Document> documents = jsonReader.get("/0/sessions");
+		assertThat(documents).isNotEmpty();
+		for (Document document : documents) {
+			assertThat(document.getText()).isNotEmpty();
+			assertThat(document.getText()).contains("Session");
+		}
+	}
+
+	@Test
+	void loadJsonObjectFromPointer() {
+		assertThat(this.ObjectResource).isNotNull();
+		JsonReader jsonReader = new JsonReader(this.ObjectResource, "name");
+		List<Document> documents = jsonReader.get("/store");
+		assertThat(documents).isNotEmpty();
+		assertThat(documents.size()).isEqualTo(1);
+		assertThat(documents.get(0).getText()).contains("name: Bike Shop");
+	}
+
+}

spring-ai-commons/src/test/java/org/springframework/ai/reader/TextReaderTests.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.transformer.splitter.TokenTextSplitter;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Christian Tzolov
+ * @author Mark Pollack
+ */
+public class TextReaderTests {
+
+	@Test
+	void loadText() {
+		Resource resource = new DefaultResourceLoader().getResource("classpath:text_source.txt");
+		assertThat(resource).isNotNull();
+		TextReader textReader = new TextReader(resource);
+		textReader.getCustomMetadata().put("customKey", "Value");
+
+		List<Document> documents0 = textReader.get();
+
+		List<Document> documents = new TokenTextSplitter().apply(documents0);
+
+		assertThat(documents.size()).isEqualTo(54);
+
+		for (Document document : documents) {
+			assertThat(document.getMetadata().get("customKey")).isEqualTo("Value");
+			assertThat(document.getMetadata().get(TextReader.SOURCE_METADATA)).isEqualTo("text_source.txt");
+			assertThat(document.getMetadata().get(TextReader.CHARSET_METADATA)).isEqualTo("UTF-8");
+			assertThat(document.getText()).isNotEmpty();
+		}
+	}
+
+	@Test
+	void loadTextFromByteArrayResource() {
+		// Test with default constructor
+		Resource defaultByteArrayResource = new ByteArrayResource("Test content".getBytes(StandardCharsets.UTF_8));
+		assertThat(defaultByteArrayResource).isNotNull();
+		TextReader defaultTextReader = new TextReader(defaultByteArrayResource);
+		defaultTextReader.getCustomMetadata().put("customKey", "DefaultValue");
+
+		List<Document> defaultDocuments = defaultTextReader.get();
+
+		assertThat(defaultDocuments).hasSize(1);
+
+		Document defaultDocument = defaultDocuments.get(0);
+		assertThat(defaultDocument.getMetadata()).containsEntry("customKey", "DefaultValue")
+			.containsEntry(TextReader.CHARSET_METADATA, "UTF-8");
+
+		// Assert on the SOURCE_METADATA for default ByteArrayResource
+		assertThat(defaultDocument.getMetadata().get(TextReader.SOURCE_METADATA))
+			.isEqualTo("Byte array resource [resource loaded from byte array]");
+
+		assertThat(defaultDocument.getText()).isEqualTo("Test content");
+
+		// Test with custom description constructor
+		String customDescription = "Custom byte array resource";
+		Resource customByteArrayResource = new ByteArrayResource(
+				"Another test content".getBytes(StandardCharsets.UTF_8), customDescription);
+		assertThat(customByteArrayResource).isNotNull();
+		TextReader customTextReader = new TextReader(customByteArrayResource);
+		customTextReader.getCustomMetadata().put("customKey", "CustomValue");
+
+		List<Document> customDocuments = customTextReader.get();
+
+		assertThat(customDocuments).hasSize(1);
+
+		Document customDocument = customDocuments.get(0);
+		assertThat(customDocument.getMetadata()).containsEntry("customKey", "CustomValue")
+			.containsEntry(TextReader.CHARSET_METADATA, "UTF-8");
+
+		// Assert on the SOURCE_METADATA for custom ByteArrayResource
+		assertThat(customDocument.getMetadata().get(TextReader.SOURCE_METADATA))
+			.isEqualTo("Byte array resource [Custom byte array resource]");
+
+		assertThat(customDocument.getText()).isEqualTo("Another test content");
+	}
+
+}

spring-ai-commons/src/test/java/org/springframework/ai/transformer/splitter/TextSplitterTests.java

@@ -0,0 +1,248 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.transformer.splitter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.document.DefaultContentFormatter;
+import org.springframework.ai.document.Document;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertAll;
+
+/**
+ * @author Christian Tzolov
+ * @author Jiwoo Kim
+ */
+public class TextSplitterTests {
+
+	static TextSplitter testTextSplitter = new TextSplitter() {
+
+		@Override
+		protected List<String> splitText(String text) {
+			int chuckSize = text.length() / 2;
+
+			List<String> chunks = new ArrayList<>();
+
+			chunks.add(text.substring(0, chuckSize));
+			chunks.add(text.substring(chuckSize));
+
+			return chunks;
+		}
+	};
+
+	@Test
+	public void testSplitText() {
+
+		var contentFormatter1 = DefaultContentFormatter.defaultConfig();
+		var contentFormatter2 = DefaultContentFormatter.defaultConfig();
+
+		assertThat(contentFormatter1).isNotSameAs(contentFormatter2);
+
+		var doc1 = new Document("In the end, writing arises when man realizes that memory is not enough.",
+				Map.of("key1", "value1", "key2", "value2"));
+		doc1.setContentFormatter(contentFormatter1);
+
+		var doc2 = new Document("The most oppressive thing about the labyrinth is that you are constantly "
+				+ "being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("key2", "value22", "key3", "value3"));
+		doc2.setContentFormatter(contentFormatter2);
+
+		List<Document> chunks = testTextSplitter.apply(List.of(doc1, doc2));
+
+		assertThat(testTextSplitter.isCopyContentFormatter()).isTrue();
+
+		assertThat(chunks).hasSize(4);
+
+		// Doc1 chunks:
+		assertThat(chunks.get(0).getText()).isEqualTo("In the end, writing arises when man");
+		assertThat(chunks.get(1).getText()).isEqualTo(" realizes that memory is not enough.");
+
+		// Doc2 chunks:
+		assertThat(chunks.get(2).getText())
+			.isEqualTo("The most oppressive thing about the labyrinth is that you are constantly being forced to ");
+		assertThat(chunks.get(3).getText())
+			.isEqualTo("choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.");
+
+		// Verify that the same, merged metadata is copied to all chunks.
+		assertThat(chunks.get(0).getMetadata()).isEqualTo(chunks.get(1).getMetadata());
+		assertThat(chunks.get(2).getMetadata()).isEqualTo(chunks.get(3).getMetadata());
+		assertThat(chunks.get(0).getMetadata()).containsKeys("key1", "key2").doesNotContainKeys("key3");
+		assertThat(chunks.get(2).getMetadata()).containsKeys("key2", "key3").doesNotContainKeys("key1");
+
+		// Verify that the content formatters are copied from the parents to the chunks.
+		// doc1 -> chunk0, chunk1 and doc2 -> chunk2, chunk3
+		assertThat(chunks.get(0).getContentFormatter()).isSameAs(contentFormatter1);
+		assertThat(chunks.get(1).getContentFormatter()).isSameAs(contentFormatter1);
+
+		assertThat(chunks.get(2).getContentFormatter()).isSameAs(contentFormatter2);
+		assertThat(chunks.get(3).getContentFormatter()).isSameAs(contentFormatter2);
+
+		// Disable copy content formatters
+		testTextSplitter.setCopyContentFormatter(false);
+		chunks = testTextSplitter.apply(List.of(doc1, doc2));
+
+		assertThat(chunks.get(0).getContentFormatter()).isNotSameAs(contentFormatter1);
+		assertThat(chunks.get(1).getContentFormatter()).isNotSameAs(contentFormatter1);
+
+		assertThat(chunks.get(2).getContentFormatter()).isNotSameAs(contentFormatter2);
+		assertThat(chunks.get(3).getContentFormatter()).isNotSameAs(contentFormatter2);
+
+	}
+
+	@Test
+	public void pageNoChunkSplit() {
+		// given
+		var doc1 = new Document("1In the end, writing arises when man realizes that memory is not enough."
+				+ "1The most oppressive thing about the labyrinth is that you are constantly "
+				+ "1being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("file_name", "sample1.pdf", "page_number", 1));
+
+		var doc2 = new Document("2In the end, writing arises when man realizes that memory is not enough."
+				+ "2The most oppressive thing about the labyrinth is that you are constantly "
+				+ "2being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("file_name", "sample1.pdf", "page_number", 2));
+
+		var doc3 = new Document("3In the end, writing arises when man realizes that memory is not enough."
+				+ "3The most oppressive thing about the labyrinth is that you are constantly "
+				+ "3being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("file_name", "sample1.pdf", "page_number", 3));
+
+		var doc4 = new Document("4In the end, writing arises when man realizes that memory is not enough."
+				+ "4The most oppressive thing about the labyrinth is that you are constantly "
+				+ "4being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("file_name", "sample1.pdf", "page_number", 4));
+
+		var tokenTextSplitter = new TokenTextSplitter();
+
+		// when
+		List<Document> splitedDocument = tokenTextSplitter.apply(List.of(doc1, doc2, doc3, doc4));
+
+		// then
+		assertAll(() -> assertThat(splitedDocument).isNotNull(), () -> assertThat(splitedDocument).isNotEmpty(),
+				() -> assertThat(splitedDocument).hasSize(4),
+				() -> assertThat(splitedDocument.get(0).getMetadata().get("page_number")).isEqualTo(1),
+				() -> assertThat(splitedDocument.get(1).getMetadata().get("page_number")).isEqualTo(2),
+				() -> assertThat(splitedDocument.get(2).getMetadata().get("page_number")).isEqualTo(3),
+				() -> assertThat(splitedDocument.get(3).getMetadata().get("page_number")).isEqualTo(4));
+	}
+
+	@Test
+	public void pageWithChunkSplit() {
+		// given
+
+		var doc1 = new Document("1In the end, writing arises when man realizes that memory is not enough."
+				+ "1The most oppressive thing about the labyrinth is that you are constantly "
+				+ "1being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("file_name", "sample1.pdf", "page_number", 1));
+
+		var doc2 = new Document(
+				"levels, their care  providers,   legal  representatives    and   families  get the  right home    and                                                                                               \n"
+						+ "                  community-based       support   and   services  at the  right time,  in the  right place.  Please   click here  to                                                                                  \n"
+						+ "                  go to Community      Living  Connections.                                                                                                                                                           \n"
+						+ "\n"
+						+ "                  I  am  trying   to  register    as  a consumer,       but   Carina    will  not   recognize      me   or  my                                                                                        \n"
+						+ "                  information.       What    should     I do?                                                                                                                                                         \n"
+						+ "\n"
+						+ "                  Please  double   check   your  form   entries  including   the spelling  of your   name   and  your                                                                                                 \n"
+						+ "                  ProviderOne     number,    or last four  digits of your  social  security  number    and   date  of birth. Please                                                                                   \n"
+						+ "                  use the  name    you  have  on  file with  the  Department     of Social  and  Health   Services   (DSHS).  Also                                                                                    \n"
+						+ "                  make   sure  you  have   a current  or  pending   assessment     with  DSHS.                                                                                                                        \n"
+						+ "\n"
+						+ "                  If  you  are  having  trouble registering,   please  contact   us or  call  us  at  1-855-796-0605.                                                                                                 \n"
+						+ "\n"
+						+ "                  The   Home     Care    Referral    Registry     has  been    absorbed       by  Consumer        Direct   Care                                                                                       \n"
+						+ "                  Network      Washington        (CDWA).       Who    can   help   me    find  care   on   Carina?                                                                                                    \n"
+						+ "\n"
+						+ "                  Consumer     Direct  Care  Network    Washington      (CDWA)    has  taken  over  from   the  Home    Care                                                                                          \n"
+						+ "                  Referral  Registry  (HCRR).   CDWA     is  responsible  for assisting  consumers     and   Individual  Providers                                                                                    \n"
+						+ "                  (IPs) to use  Carina  to find  matches.    CDWA    staff are  available  across   the state  to  assist                                                                                             \n"
+						+ "                  consumers    to  sign up  in the  Carina  system    and  help  IPs get  (re)contracted    or hired  to work.                                                                                        \n"
+						+ "\n"
+						+ "                  What    are   some    good     interview     questions      I should    ask   providers?                                                                                                            \n"
+						+ "\n"
+						+ "                  Your  approach    to the  interview   is important,   you   are offering   a job to  someone    who   is looking                                                                                    \n"
+						+ "                  for work.  The  person    you  interview   may   be  nervous.   Put  them   at ease,  call them   by their  first                                                                                   \n"
+						+ "                  name,   maintain   eye  contact   and  tell them   a little  about yourself.   Read  more    tips and  specific                                                                                     \n"
+						+ "                  interview   questions   in our  blog:  What   to Ask  Potential   Providers.                                                                                                                        \n"
+						+ "\n"
+						+ "                  I  am  ready    to  hire  a  home     care   provider!                                                                                                                                              \n"
+						+ "\n"
+						+ "                  You  found   an Individual   Provider   (IP) that  you  would   like to hire?  That  is exciting!  In order  for                                                                                    \n"
+						+ "                  them   to start working,   contact   Consumer     Direct  Care   Network    Washington     (CDWA)     and  request                                                                                  \n"
+						+ "                  authorization.   They   cannot   start work   before   you  have   received   an  Okay  to  Work   from   CDWA.                                                                                     \n"
+						+ "\n"
+						+ "                  Consumers     should   continue   to work   with  their  case  manager,    who   will help  you  create  a  Plan of                                                                                 \n"
+						+ "                  Care  and  access   needed   services.\n"
+						+ "Once   you  have  decided    on an  IP to work   with,  they  should\n" + "\n",
+				Map.of("file_name", "sample1.pdf", "page_number", 2));
+
+		var doc3 = new Document("3In the end, writing arises when man realizes that memory is not enough."
+				+ "3The most oppressive thing about the labyrinth is that you are constantly "
+				+ "3being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("file_name", "sample1.pdf", "page_number", 3));
+
+		var tokenTextSplitter = new TokenTextSplitter();
+
+		// when
+		List<Document> splitedDocument = tokenTextSplitter.apply(List.of(doc1, doc2, doc3));
+
+		// then
+		assertAll(() -> assertThat(splitedDocument).isNotNull(), () -> assertThat(splitedDocument).isNotEmpty(),
+				() -> assertThat(splitedDocument).hasSize(4),
+				() -> assertThat(splitedDocument.get(0).getMetadata().get("page_number")).isEqualTo(1),
+				() -> assertThat(splitedDocument.get(1).getMetadata().get("page_number")).isEqualTo(2),
+				() -> assertThat(splitedDocument.get(2).getMetadata().get("page_number")).isEqualTo(2),
+				() -> assertThat(splitedDocument.get(3).getMetadata().get("page_number")).isEqualTo(3));
+	}
+
+	@Test
+	public void testSplitTextWithNullMetadata() {
+
+		var contentFormatter = DefaultContentFormatter.defaultConfig();
+
+		var doc = new Document("In the end, writing arises when man realizes that memory is not enough.");
+
+		doc.getMetadata().put("key1", "value1");
+		doc.getMetadata().put("key2", null);
+
+		doc.setContentFormatter(contentFormatter);
+
+		List<Document> chunks = testTextSplitter.apply(List.of(doc));
+
+		assertThat(testTextSplitter.isCopyContentFormatter()).isTrue();
+
+		assertThat(chunks).hasSize(2);
+
+		// Doc chunks:
+		assertThat(chunks.get(0).getText()).isEqualTo("In the end, writing arises when man");
+		assertThat(chunks.get(1).getText()).isEqualTo(" realizes that memory is not enough.");
+
+		// Verify that the same, merged metadata is copied to all chunks.
+		assertThat(chunks.get(0).getMetadata()).isEqualTo(chunks.get(1).getMetadata());
+		assertThat(chunks.get(1).getMetadata()).containsKeys("key1");
+
+		// Verify that the content formatters are copied from the parents to the chunks.
+		assertThat(chunks.get(0).getContentFormatter()).isSameAs(contentFormatter);
+		assertThat(chunks.get(1).getContentFormatter()).isSameAs(contentFormatter);
+	}
+
+}

spring-ai-commons/src/test/java/org/springframework/ai/transformer/splitter/TokenTextSplitterTest.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.transformer.splitter;
+
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.document.DefaultContentFormatter;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.transformer.splitter.TokenTextSplitter;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Ricken Bazolo
+ */
+public class TokenTextSplitterTest {
+
+	@Test
+	public void testTokenTextSplitterBuilderWithDefaultValues() {
+
+		var contentFormatter1 = DefaultContentFormatter.defaultConfig();
+		var contentFormatter2 = DefaultContentFormatter.defaultConfig();
+
+		assertThat(contentFormatter1).isNotSameAs(contentFormatter2);
+
+		var doc1 = new Document("In the end, writing arises when man realizes that memory is not enough.",
+				Map.of("key1", "value1", "key2", "value2"));
+		doc1.setContentFormatter(contentFormatter1);
+
+		var doc2 = new Document("The most oppressive thing about the labyrinth is that you are constantly "
+				+ "being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("key2", "value22", "key3", "value3"));
+		doc2.setContentFormatter(contentFormatter2);
+
+		var tokenTextSplitter = new TokenTextSplitter();
+
+		var chunks = tokenTextSplitter.apply(List.of(doc1, doc2));
+
+		assertThat(chunks.size()).isEqualTo(2);
+
+		// Doc 1
+		assertThat(chunks.get(0).getText())
+			.isEqualTo("In the end, writing arises when man realizes that memory is not enough.");
+		// Doc 2
+		assertThat(chunks.get(1).getText()).isEqualTo(
+				"The most oppressive thing about the labyrinth is that you are constantly being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.");
+
+		assertThat(chunks.get(0).getMetadata()).containsKeys("key1", "key2").doesNotContainKeys("key3");
+		assertThat(chunks.get(1).getMetadata()).containsKeys("key2", "key3").doesNotContainKeys("key1");
+	}
+
+	@Test
+	public void testTokenTextSplitterBuilderWithAllFields() {
+
+		var contentFormatter1 = DefaultContentFormatter.defaultConfig();
+		var contentFormatter2 = DefaultContentFormatter.defaultConfig();
+
+		assertThat(contentFormatter1).isNotSameAs(contentFormatter2);
+
+		var doc1 = new Document("In the end, writing arises when man realizes that memory is not enough.",
+				Map.of("key1", "value1", "key2", "value2"));
+		doc1.setContentFormatter(contentFormatter1);
+
+		var doc2 = new Document("The most oppressive thing about the labyrinth is that you are constantly "
+				+ "being forced to choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.",
+				Map.of("key2", "value22", "key3", "value3"));
+		doc2.setContentFormatter(contentFormatter2);
+
+		var tokenTextSplitter = TokenTextSplitter.builder()
+			.withChunkSize(10)
+			.withMinChunkSizeChars(5)
+			.withMinChunkLengthToEmbed(3)
+			.withMaxNumChunks(50)
+			.withKeepSeparator(true)
+			.build();
+
+		var chunks = tokenTextSplitter.apply(List.of(doc1, doc2));
+
+		assertThat(chunks.size()).isEqualTo(6);
+
+		// Doc 1
+		assertThat(chunks.get(0).getText()).isEqualTo("In the end, writing arises when man realizes that");
+		assertThat(chunks.get(1).getText()).isEqualTo("memory is not enough.");
+
+		// Doc 2
+		assertThat(chunks.get(2).getText()).isEqualTo("The most oppressive thing about the labyrinth is that you");
+		assertThat(chunks.get(3).getText()).isEqualTo("are constantly being forced to choose.");
+		assertThat(chunks.get(4).getText()).isEqualTo("It isn’t the lack of an exit, but");
+		assertThat(chunks.get(5).getText()).isEqualTo("the abundance of exits that is so disorienting");
+
+		// Verify that the same, merged metadata is copied to all chunks.
+		assertThat(chunks.get(0).getMetadata()).isEqualTo(chunks.get(1).getMetadata());
+		assertThat(chunks.get(2).getMetadata()).isEqualTo(chunks.get(3).getMetadata());
+
+		assertThat(chunks.get(0).getMetadata()).containsKeys("key1", "key2").doesNotContainKeys("key3");
+		assertThat(chunks.get(2).getMetadata()).containsKeys("key2", "key3").doesNotContainKeys("key1");
+	}
+
+}

spring-ai-commons/src/test/resources/events.json

@@ -0,0 +1,15 @@
+[
+  {
+    "sessions": [
+      {
+        "description": "Session one"
+      },
+      {
+        "description": "Session two"
+      },
+      {
+        "description": "Session three"
+      }
+    ]
+  }
+]