Apache Tika based reader for all kinds of documents

- Provides a rudimentary text extractions for multitude of document formats,
   including PDF, Word Doc/Docx PowerPoint ppt/pptx and many more.
 - Generates a single Document for the extracted text.
 - No pre or post processing and cleansing for the text.
 - Move the ExtractedTextFormatter from pdf reader to the core reader to enable reusability. Improve the tika reader

document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/config/PdfDocumentReaderConfig.java

@@ -1,8 +1,8 @@
 package org.springframework.ai.reader.pdf.config;
 
+import org.springframework.ai.reader.ExtractedTextFormatter;
 import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
 import org.springframework.ai.reader.pdf.ParagraphPdfDocumentReader;
-import org.springframework.ai.reader.pdf.layout.PageExtractedTextFormatter;
 import org.springframework.util.Assert;
 
 /**
@@ -23,7 +23,7 @@ public class PdfDocumentReaderConfig {
 
 	public final int pageBottomMargin;
 
-	public final PageExtractedTextFormatter pageExtractedTextFormatter;
+	public final ExtractedTextFormatter pageExtractedTextFormatter;
 
 	/**
 	 * Start building a new configuration.
@@ -57,7 +57,7 @@ public class PdfDocumentReaderConfig {
 
 		private int pageBottomMargin = 0;
 
-		private PageExtractedTextFormatter pageExtractedTextFormatter = PageExtractedTextFormatter.defaults();
+		private ExtractedTextFormatter pageExtractedTextFormatter = ExtractedTextFormatter.defaults();
 
 		private boolean reversedParagraphPosition = false;
 
@@ -70,7 +70,7 @@ public class PdfDocumentReaderConfig {
 		 * @return this builder
 		 */
 		public PdfDocumentReaderConfig.Builder withPageExtractedTextFormatter(
-				PageExtractedTextFormatter pageExtractedTextFormatter) {
+				ExtractedTextFormatter pageExtractedTextFormatter) {
 			Assert.notNull(pagesPerDocument >= 0, "PageExtractedTextFormatter must not be null.");
 			this.pageExtractedTextFormatter = pageExtractedTextFormatter;
 			return this;

document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/PagePdfDocumentReaderTests.java

@@ -22,8 +22,8 @@ import java.util.stream.Collectors;
 import org.junit.jupiter.api.Test;
 
 import org.springframework.ai.document.Document;
+import org.springframework.ai.reader.ExtractedTextFormatter;
 import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig;
-import org.springframework.ai.reader.pdf.layout.PageExtractedTextFormatter;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -39,7 +39,7 @@ public class PagePdfDocumentReaderTests {
 				PdfDocumentReaderConfig.builder()
 					.withPageTopMargin(0)
 					.withPageBottomMargin(0)
-					.withPageExtractedTextFormatter(PageExtractedTextFormatter.builder()
+					.withPageExtractedTextFormatter(ExtractedTextFormatter.builder()
 						.withNumberOfTopTextLinesToDelete(0)
 						.withNumberOfBottomTextLinesToDelete(3)
 						.withNumberOfTopPagesToSkipBeforeDelete(0)

document-readers/tika-reader/pom.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.springframework.experimental.ai</groupId>
+		<artifactId>spring-ai</artifactId>
+		<version>0.7.0-SNAPSHOT</version>
+		<relativePath>../../pom.xml</relativePath>
+	</parent>
+	<artifactId>spring-ai-tika-document-reader</artifactId>
+	<packaging>jar</packaging>
+	<name>Spring AI Document Reader - Tika</name>
+	<description>Spring AI Tika document reader</description>
+	<url>https://github.com/spring-projects-experimental/spring-ai</url>
+
+	<scm>
+		<url>https://github.com/spring-projects-experimental/spring-ai</url>
+		<connection>git://github.com/spring-projects-experimental/spring-ai.git</connection>
+		<developerConnection>git@github.com:spring-projects-experimental/spring-ai.git</developerConnection>
+	</scm>
+
+	<properties>
+		<tika.version>2.9.0</tika.version>
+	</properties>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.springframework.experimental.ai</groupId>
+			<artifactId>spring-ai-core</artifactId>
+			<version>${parent.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.tika</groupId>
+			<artifactId>tika-core</artifactId>
+			<version>${tika.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.tika</groupId>
+			<artifactId>tika-parsers-standard-package</artifactId>
+			<version>${tika.version}</version>
+		</dependency>
+
+
+		<!-- TESTING -->
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-test</artifactId>
+			<scope>test</scope>
+		</dependency>
+
+	</dependencies>
+
+</project>

document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2023-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader.tika;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentReader;
+import org.springframework.ai.reader.ExtractedTextFormatter;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
+import org.springframework.util.StringUtils;
+
+/**
+ * A document reader that leverages Apache Tika to extract text from a variety of document
+ * formats, such as PDF, DOC/DOCX, PPT/PPTX, and HTML. For a comprehensive list of
+ * supported formats, refer to: https://tika.apache.org/2.9.0/formats.html.
+ *
+ * This reader directly provides the extracted text without any additional formatting. All
+ * extracted texts are encapsulated within a {@link Document} instance.
+ *
+ * If you require more specialized handling for PDFs, consider using the
+ * PagePdfDocumentReader or ParagraphPdfDocumentReader.
+ *
+ * @author Christian Tzolov
+ */
+
+public class TikaDocumentReader implements DocumentReader {
+
+	/**
+	 * Metadata key representing the source of the document.
+	 */
+	public static final String METADATA_SOURCE = "source";
+
+	/**
+	 * Parser to automatically detect the type of document and extract text.
+	 */
+	private final AutoDetectParser parser;
+
+	/**
+	 * Handler to manage content extraction.
+	 */
+	private final ContentHandler handler;
+
+	/**
+	 * Metadata associated with the document being read.
+	 */
+	private final Metadata metadata;
+
+	/**
+	 * Parsing context containing information about the parsing process.
+	 */
+	private final ParseContext context;
+
+	/**
+	 * The resource pointing to the document.
+	 */
+	private final Resource resource;
+
+	/**
+	 * Formatter for the extracted text.
+	 */
+	private final ExtractedTextFormatter textFormatter;
+
+	/**
+	 * Constructor initializing the reader with a given resource URL.
+	 * @param resourceUrl URL to the resource
+	 */
+	public TikaDocumentReader(String resourceUrl) {
+		this(resourceUrl, ExtractedTextFormatter.defaults());
+	}
+
+	/**
+	 * Constructor initializing the reader with a given resource URL and a text formatter.
+	 * @param resourceUrl URL to the resource
+	 * @param textFormatter Formatter for the extracted text
+	 */
+	public TikaDocumentReader(String resourceUrl, ExtractedTextFormatter textFormatter) {
+		this(new DefaultResourceLoader().getResource(resourceUrl), textFormatter);
+	}
+
+	/**
+	 * Constructor initializing the reader with a resource.
+	 * @param resource Resource pointing to the document
+	 */
+	public TikaDocumentReader(Resource resource) {
+		this(resource, ExtractedTextFormatter.defaults());
+	}
+
+	/**
+	 * Constructor initializing the reader with a resource and a text formatter.
+	 * @param resource Resource pointing to the document
+	 * @param textFormatter Formatter for the extracted text
+	 */
+	public TikaDocumentReader(Resource resource, ExtractedTextFormatter textFormatter) {
+		this(resource, new BodyContentHandler(), textFormatter);
+	}
+
+	/**
+	 * Constructor initializing the reader with a resource, content handler, and a text
+	 * formatter.
+	 * @param resource Resource pointing to the document
+	 * @param contentHandler Handler to manage content extraction
+	 * @param textFormatter Formatter for the extracted text
+	 */
+	public TikaDocumentReader(Resource resource, ContentHandler contentHandler, ExtractedTextFormatter textFormatter) {
+		this.parser = new AutoDetectParser();
+		this.handler = contentHandler;
+		this.metadata = new Metadata();
+		this.context = new ParseContext();
+		this.resource = resource;
+		this.textFormatter = textFormatter;
+	}
+
+	/**
+	 * Extracts and returns the list of documents from the resource.
+	 * @return List of extracted {@link Document}
+	 */
+	@Override
+	public List<Document> get() {
+		try (InputStream stream = this.resource.getInputStream()) {
+			this.parser.parse(stream, this.handler, this.metadata, this.context);
+			return List.of(toDocument(this.handler.toString()));
+		}
+		catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	/**
+	 * Converts the given text to a {@link Document}.
+	 * @param docText Text to be converted
+	 * @return Converted document
+	 */
+	private Document toDocument(String docText) {
+		docText = Objects.requireNonNullElse(docText, "");
+		docText = this.textFormatter.format(docText);
+		Document doc = new Document(docText);
+		doc.getMetadata().put(METADATA_SOURCE, resourceName());
+		return doc;
+	}
+
+	/**
+	 * Returns the name of the resource. If the filename is not present, it returns the
+	 * URI of the resource.
+	 * @return Name or URI of the resource
+	 */
+	private String resourceName() {
+		try {
+			var resourceName = this.resource.getFilename();
+			if (!StringUtils.hasText(resourceName)) {
+				resourceName = this.resource.getURI().toString();
+			}
+			return resourceName;
+		}
+		catch (IOException e) {
+			return String.format("Invalid source URI: %s", e.getMessage());
+		}
+	}
+
+}

document-readers/tika-reader/src/test/java/org/springframework/ai/reader/tika/TikaDocumentReaderTests.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2023-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.reader.tika;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Christian Tzolov
+ */
+public class TikaDocumentReaderTests {
+
+	@ParameterizedTest
+	@CsvSource({
+			"classpath:/word-sample.docx,word-sample.docx,Two kinds of links are possible, those that refer to an external website",
+			"classpath:/word-sample.doc,word-sample.doc,The limited permissions granted above are perpetual and will not be revoked by OASIS",
+			"classpath:/sample2.pdf,sample2.pdf,Consult doc/pdftex/manual.pdf from your tetex distribution for more",
+			"classpath:/sample.ppt,sample.ppt,Sed ipsum tortor, fringilla a consectetur eget, cursus posuere sem.",
+			"classpath:/sample.pptx,sample.pptx,Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
+			"https://docs.spring.io/spring-ai/reference/,https://docs.spring.io/spring-ai/reference/,help set up essential dependencies and classes." })
+	public void testDocx(String resourceUri, String resourceName, String contentSnipped) {
+
+		var docs = new TikaDocumentReader(resourceUri).get();
+		assertThat(docs).hasSize(1);
+
+		var doc = docs.get(0);
+
+		assertThat(doc.getMetadata()).containsKeys(TikaDocumentReader.METADATA_SOURCE);
+		assertThat(doc.getMetadata().get(TikaDocumentReader.METADATA_SOURCE)).isEqualTo(resourceName);
+		assertThat(doc.getContent()).contains(contentSnipped);
+	}
+
+}

pom.xml

@@ -25,6 +25,7 @@
 		<module>vector-stores/spring-ai-neo4j-store</module>
 		<module>embedding-clients/spring-ai-postgresml-embedding-client</module>
 		<module>document-readers/pdf-reader</module>
+		<module>document-readers/tika-reader</module>
 	</modules>
 
 	<organization>

spring-ai-core/src/main/java/org/springframework/ai/reader/ExtractedTextFormatter.java

@@ -14,45 +14,82 @@
  * limitations under the License.
  */
 
-package org.springframework.ai.reader.pdf.layout;
+package org.springframework.ai.reader;
 
 import org.springframework.util.StringUtils;
 
 /**
- * Provides text formatting options for extracted PDF page text, including left alignment
- * and the ability to trim and delete lines from the top and bottom of the text.
+ * A utility to reformat extracted text content before encapsulating it in a
+ * {@link org.springframework.ai.document.Document}. This formatter provides the following
+ * functionalities:
  *
- * This class allows customization of text formatting applied to extracted PDF page text.
- * It can align the text to the left, remove specified lines from the top and bottom of
- * the text, and trim adjacent blank lines.
+ * <ul>
+ * <li>Left alignment of text</li>
+ * <li>Removal of specified lines from the beginning and end of content</li>
+ * <li>Consolidation of consecutive blank lines</li>
+ * </ul>
+ *
+ * An instance of this formatter can be customized using the {@link Builder} nested class.
  *
  * @author Christian Tzolov
  */
-public class PageExtractedTextFormatter {
+public class ExtractedTextFormatter {
 
+	/** Flag indicating if the text should be left-aligned */
 	private boolean leftAlignment;
 
+	/** Number of top pages to skip before performing delete operations */
 	private int numberOfTopPagesToSkipBeforeDelete;
 
+	/** Number of top text lines to delete from a page */
 	private int numberOfTopTextLinesToDelete;
 
+	/** Number of bottom text lines to delete from a page */
 	private int numberOfBottomTextLinesToDelete;
 
-	private PageExtractedTextFormatter(Builder builder) {
+	/**
+	 * Private constructor to initialize the formatter from the builder.
+	 * @param builder Builder used to initialize the formatter.
+	 */
+	private ExtractedTextFormatter(Builder builder) {
 		this.leftAlignment = builder.leftAlignment;
 		this.numberOfBottomTextLinesToDelete = builder.numberOfBottomTextLinesToDelete;
 		this.numberOfTopPagesToSkipBeforeDelete = builder.numberOfTopPagesToSkipBeforeDelete;
 		this.numberOfTopTextLinesToDelete = builder.numberOfTopTextLinesToDelete;
 	}
 
+	/**
+	 * Provides an instance of the builder for this formatter.
+	 * @return an instance of the builder.
+	 */
 	public static Builder builder() {
 		return new Builder();
 	}
 
-	public static PageExtractedTextFormatter defaults() {
+	/**
+	 * Provides a default instance of the formatter.
+	 * @return default instance of the formatter.
+	 */
+	public static ExtractedTextFormatter defaults() {
 		return new Builder().build();
 	}
 
+	/**
+	 * Formats the provided text according to the formatter's configuration.
+	 * @param pageText Text to be formatted.
+	 * @return Formatted text.
+	 */
+	public String format(String pageText) {
+		return this.format(pageText, 0);
+	}
+
+	/**
+	 * Formats the provided text based on the formatter's configuration, considering the
+	 * page number.
+	 * @param pageText Text to be formatted.
+	 * @param pageNumber Page number of the provided text.
+	 * @return Formatted text.
+	 */
 	public String format(String pageText, int pageNumber) {
 
 		var text = trimAdjacentBlankLines(pageText);
@@ -69,6 +106,36 @@ public class PageExtractedTextFormatter {
 		return text;
 	}
 
+	/**
+	 * The {@code Builder} class is a nested static class of
+	 * {@link ExtractedTextFormatter} designed to facilitate the creation and
+	 * customization of instances of {@link ExtractedTextFormatter}.
+	 *
+	 * <p>
+	 * It allows for a step-by-step, fluent construction of the
+	 * {@link ExtractedTextFormatter}, by providing methods to set specific configurations
+	 * such as left alignment of text, the number of top lines or bottom lines to delete,
+	 * and the number of top pages to skip before deletion. Each configuration method in
+	 * the builder returns the builder instance itself, enabling method chaining.
+	 * </p>
+	 *
+	 *
+	 * By default, the builder sets:
+	 * <ul>
+	 * <li>Left alignment to {@code false}</li>
+	 * <li>Number of top pages to skip before deletion to 0</li>
+	 * <li>Number of top text lines to delete to 0</li>
+	 * <li>Number of bottom text lines to delete to 0</li>
+	 * </ul>
+	 * 
+	 *
+	 * <p>
+	 * After configuring the builder, calling the {@link #build()} method will return a
+	 * new instance of {@link ExtractedTextFormatter} with the specified configurations.
+	 * </p>
+	 *
+	 * @see ExtractedTextFormatter
+	 */
 	public static class Builder {
 
 		private boolean leftAlignment = false;
@@ -120,8 +187,25 @@ public class PageExtractedTextFormatter {
 			return this;
 		}
 
-		public PageExtractedTextFormatter build() {
-			return new PageExtractedTextFormatter(this);
+		/**
+		 * Constructs and returns an instance of {@link ExtractedTextFormatter} using the
+		 * configurations set on this builder.
+		 *
+		 * <p>
+		 * This method uses the values set on the builder to initialize the configuration
+		 * for the {@link ExtractedTextFormatter} instance. If no values are explicitly
+		 * set on the builder, the defaults specified in the builder are used.
+		 * </p>
+		 *
+		 * <p>
+		 * It's recommended to use this method only once per builder instance to ensure
+		 * that each {@link ExtractedTextFormatter} object is configured as intended.
+		 * </p>
+		 * @return a new instance of {@link ExtractedTextFormatter} configured with the
+		 * values set on this builder.
+		 */
+		public ExtractedTextFormatter build() {
+			return new ExtractedTextFormatter(this);
 		}
 
 	}
@@ -132,9 +216,7 @@ public class PageExtractedTextFormatter {
 	 * @return Returns the same text but with blank lines trimmed.
 	 */
 	public static String trimAdjacentBlankLines(String pageText) {
-
 		return pageText.replaceAll("(?m)(^ *\n)", "\n").replaceAll("(?m)^$([\r\n]+?)(^$[\r\n]+?^)+", "$1");
-
 	}
 
 	/**
@@ -167,6 +249,27 @@ public class PageExtractedTextFormatter {
 		return pageText.substring(0, truncateIndex);
 	}
 
+	/**
+	 * Removes a specified number of lines from the top part of the given text.
+	 *
+	 * <p>
+	 * This method takes a text and trims it by removing a certain number of lines from
+	 * the top. If the provided text is null or contains only whitespace, it will be
+	 * returned as is. If the number of lines to remove exceeds the actual number of lines
+	 * in the text, the result will be an empty string.
+	 * </p>
+	 *
+	 * <p>
+	 * The method identifies lines based on the system's line separator, making it
+	 * compatible with different platforms.
+	 * </p>
+	 * @param pageText The text from which the top lines need to be removed. If this is
+	 * null, empty, or consists only of whitespace, it will be returned unchanged.
+	 * @param numberOfLines The number of lines to remove from the top of the text. If
+	 * this exceeds the actual number of lines in the text, an empty string will be
+	 * returned.
+	 * @return The text with the specified number of lines removed from the top.
+	 */
 	public static String deleteTopTextLines(String pageText, int numberOfLines) {
 		if (!StringUtils.hasText(pageText)) {
 			return pageText;