From 8784a59673ba513afdc182343eea6b7b4faab745 Mon Sep 17 00:00:00 2001 From: Mark Pollack Date: Thu, 7 Mar 2024 10:14:30 -0500 Subject: [PATCH] Add logging of page processing progress in PagePdfDocumentReader --- .../ai/reader/pdf/PagePdfDocumentReader.java | 16 +++++++++++++++- .../reader/pdf/ParagraphPdfDocumentReader.java | 7 ++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/PagePdfDocumentReader.java b/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/PagePdfDocumentReader.java index b3ec125b5..6b48ecdb7 100644 --- a/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/PagePdfDocumentReader.java +++ b/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/PagePdfDocumentReader.java @@ -25,6 +25,8 @@ import org.apache.pdfbox.pdfparser.PDFParser; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.document.DocumentReader; import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig; @@ -44,6 +46,8 @@ import org.springframework.util.StringUtils; */ public class PagePdfDocumentReader implements DocumentReader { + private final Logger logger = LoggerFactory.getLogger(getClass()); + private static final String PDF_PAGE_REGION = "pdfPageRegion"; public static final String METADATA_START_PAGE_NUMBER = "page_number"; @@ -99,7 +103,17 @@ public class PagePdfDocumentReader implements DocumentReader { List pageTextGroupList = new ArrayList<>(); + int totalPages = this.document.getDocumentCatalog().getPages().getCount(); + int logFrequency = totalPages > 10 ? totalPages / 10 : 1; // if less than 10 + // pages, print + // each iteration + int counter = 0; + for (PDPage page : this.document.getDocumentCatalog().getPages()) { + if (counter % logFrequency == 0 && counter / logFrequency < 10) { + logger.info("Processing PDF page: {}", (counter + 1)); + } + counter++; pagesPerDocument++; @@ -139,7 +153,7 @@ public class PagePdfDocumentReader implements DocumentReader { readDocuments.add(toDocument(pageTextGroupList.stream().collect(Collectors.joining()), startPageNumber, pageNumber)); } - + logger.info("Processing {} pages", totalPages); return readDocuments; } diff --git a/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReader.java b/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReader.java index 31331aa8a..3d5235bf1 100644 --- a/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReader.java +++ b/document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReader.java @@ -23,6 +23,8 @@ import java.util.List; import org.apache.pdfbox.pdfparser.PDFParser; import org.apache.pdfbox.pdmodel.PDDocument; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.document.DocumentReader; import org.springframework.ai.reader.pdf.config.ParagraphManager; @@ -46,6 +48,8 @@ import org.springframework.util.StringUtils; */ public class ParagraphPdfDocumentReader implements DocumentReader { + private final Logger logger = LoggerFactory.getLogger(getClass()); + // Constants for metadata keys private static final String METADATA_START_PAGE = "page_number"; @@ -128,6 +132,7 @@ public class ParagraphPdfDocumentReader implements DocumentReader { List documents = new ArrayList<>(paragraphs.size()); if (!CollectionUtils.isEmpty(paragraphs)) { + logger.info("Start processing paragraphs from PDF"); Iterator itr = paragraphs.iterator(); var current = itr.next(); @@ -146,7 +151,7 @@ public class ParagraphPdfDocumentReader implements DocumentReader { } } } - + logger.info("End processing paragraphs from PDF"); return documents; }