Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add logging of page processing progress in PagePdfDocumentReader

Browse Source
This commit is contained in:
Mark Pollack
2024-03-07 10:14:30 -05:00
parent 7f60e03054
commit 8784a59673
2 changed files with 21 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/PagePdfDocumentReader.java
View File

@@ -25,6 +25,8 @@ import org.apache.pdfbox.pdfparser.PDFParser;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.document.DocumentReader;
import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig;
@@ -44,6 +46,8 @@ import org.springframework.util.StringUtils;
*/
public class PagePdfDocumentReader implements DocumentReader {
private final Logger logger = LoggerFactory.getLogger(getClass());
private static final String PDF_PAGE_REGION = "pdfPageRegion";
public static final String METADATA_START_PAGE_NUMBER = "page_number";
@@ -99,7 +103,17 @@ public class PagePdfDocumentReader implements DocumentReader {
List<String> pageTextGroupList = new ArrayList<>();
int totalPages = this.document.getDocumentCatalog().getPages().getCount();
int logFrequency = totalPages > 10 ? totalPages / 10 : 1; // if less than 10
// pages, print
// each iteration
int counter = 0;
for (PDPage page : this.document.getDocumentCatalog().getPages()) {
if (counter % logFrequency == 0 && counter / logFrequency < 10) {
logger.info("Processing PDF page: {}", (counter + 1));
}
counter++;
pagesPerDocument++;
@@ -139,7 +153,7 @@ public class PagePdfDocumentReader implements DocumentReader {
readDocuments.add(toDocument(pageTextGroupList.stream().collect(Collectors.joining()), startPageNumber,
pageNumber));
}
logger.info("Processing {} pages", totalPages);
return readDocuments;
}

7
document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReader.java
View File

@@ -23,6 +23,8 @@ import java.util.List;
import org.apache.pdfbox.pdfparser.PDFParser;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.document.DocumentReader;
import org.springframework.ai.reader.pdf.config.ParagraphManager;
@@ -46,6 +48,8 @@ import org.springframework.util.StringUtils;
*/
public class ParagraphPdfDocumentReader implements DocumentReader {
private final Logger logger = LoggerFactory.getLogger(getClass());
// Constants for metadata keys
private static final String METADATA_START_PAGE = "page_number";
@@ -128,6 +132,7 @@ public class ParagraphPdfDocumentReader implements DocumentReader {
List<Document> documents = new ArrayList<>(paragraphs.size());
if (!CollectionUtils.isEmpty(paragraphs)) {
logger.info("Start processing paragraphs from PDF");
Iterator<Paragraph> itr = paragraphs.iterator();
var current = itr.next();
@@ -146,7 +151,7 @@ public class ParagraphPdfDocumentReader implements DocumentReader {
}
}
}
logger.info("End processing paragraphs from PDF");
return documents;
}