From 0657e1bab512118929160a2bdccfa5f80da005ec Mon Sep 17 00:00:00 2001 From: Mark Pollack Date: Wed, 20 Dec 2023 16:13:00 -0500 Subject: [PATCH] Set writelimit of default BodyContentHandler to unlimited in TikaDocumentReader --- .../springframework/ai/reader/tika/TikaDocumentReader.java | 6 ++++-- .../main/antora/modules/ROOT/pages/api/etl-pipeline.adoc | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java b/document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java index 6f1fb15b9..46128d6ee 100644 --- a/document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java +++ b/document-readers/tika-reader/src/main/java/org/springframework/ai/reader/tika/TikaDocumentReader.java @@ -111,12 +111,14 @@ public class TikaDocumentReader implements DocumentReader { } /** - * Constructor initializing the reader with a resource and a text formatter. + * Constructor initializing the reader with a resource and a text formatter. This + * constructor will create a BodyContentHandler that allows for reading large PDFs + * (constrained only by memory) * @param resource Resource pointing to the document * @param textFormatter Formatter for the extracted text */ public TikaDocumentReader(Resource resource, ExtractedTextFormatter textFormatter) { - this(resource, new BodyContentHandler(), textFormatter); + this(resource, new BodyContentHandler(-1), textFormatter); } /** diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/etl-pipeline.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/etl-pipeline.adoc index 2893e019a..c40358b04 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/etl-pipeline.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/etl-pipeline.adoc @@ -4,7 +4,7 @@ The Extraction, Transformation, and Loading (ETL) framework serves as the backbo The ETL pipeline orchestrates the flow from raw data sources to a structured vector store, ensuring data is in the optimal format for retrieval by the AI model. -The RAG use-case is designed to augment the capabilities of generative models by retrieving relevant information from a body of data to enhancing the quality and relevance of the generated output. +The RAG use-case is text to augment the capabilities of generative models by retrieving relevant information from a body of data to enhancing the quality and relevance of the generated output. == API Overview