Add Markdown document reader with enhanced features
This commit introduces a new Markdown document reader with several key features and improvements: * Add support for text with various formatting elements * Implement handling for horizontal rules and hard line breaks * Add functionality for inline and block code sections * Incorporate blockquote handling * Support ordered and unordered lists * Introduce additional metadata capabilities * Include JavaDocs Update ETL documentation to reflect these new features and usage. Fixes #105
This commit is contained in:
Piotr Olaszewski
committed by
Mark Pollack
Mark Pollack
parent
a0ee10fa91
commit
56e678c487
@@ -0,0 +1,230 @@
|
||||
package org.springframework.ai.reader.markdown;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.groups.Tuple.tuple;
|
||||
|
||||
/**
|
||||
* @author Piotr Olaszewski
|
||||
*/
|
||||
class MarkdownDocumentReaderTest {
|
||||
|
||||
@Test
|
||||
void testOnlyHeadersWithParagraphs() {
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/only-headers.md");
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(4)
|
||||
.extracting(Document::getMetadata, Document::getContent)
|
||||
.containsOnly(tuple(Map.of("category", "header_1", "title", "Header 1a"),
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue."),
|
||||
tuple(Map.of("category", "header_1", "title", "Header 1b"),
|
||||
"Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam lobortis risus libero, sed sollicitudin risus cursus in. Morbi enim metus, ornare vel lacinia eget, venenatis vel nibh."),
|
||||
tuple(Map.of("category", "header_2", "title", "Header 2b"),
|
||||
"Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero."),
|
||||
tuple(Map.of("category", "header_2", "title", "Header 2c"),
|
||||
"Ut rhoncus nec justo a porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit."));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWithFormatting() {
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/with-formatting.md");
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(2)
|
||||
.extracting(Document::getMetadata, Document::getContent)
|
||||
.containsOnly(tuple(Map.of("category", "header_1", "title", "This is a fancy header name"),
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida. Cras accumsan tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu elementum dignissim."),
|
||||
tuple(Map.of("category", "header_3", "title", "Header 3"),
|
||||
"Aenean eu leo eu nibh tristique posuere quis quis massa."));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDocumentDividedViaHorizontalRules() {
|
||||
MarkdownDocumentReaderConfig config = MarkdownDocumentReaderConfig.builder()
|
||||
.withHorizontalRuleCreateDocument(true)
|
||||
.build();
|
||||
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/horizontal-rules.md", config);
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(7)
|
||||
.extracting(Document::getMetadata, Document::getContent)
|
||||
.containsOnly(tuple(Map.of(),
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida."),
|
||||
tuple(Map.of(),
|
||||
"Cras accumsan tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu elementum dignissim."),
|
||||
tuple(Map.of(),
|
||||
"Nullam nisi dui, egestas nec sem nec, interdum lobortis enim. Pellentesque odio orci, faucibus eu luctus nec, venenatis et magna."),
|
||||
tuple(Map.of(),
|
||||
"Vestibulum nec eros non felis fermentum posuere eget ac risus. Curabitur et fringilla massa. Cras facilisis nec nisl sit amet sagittis."),
|
||||
tuple(Map.of(),
|
||||
"Aenean eu leo eu nibh tristique posuere quis quis massa. Nullam lacinia luctus sem ut vehicula."),
|
||||
tuple(Map.of(),
|
||||
"Aenean quis vulputate mi. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Nam tincidunt nunc a tortor tincidunt, nec lobortis diam rhoncus."),
|
||||
tuple(Map.of(), "Nulla facilisi. Phasellus eget tellus sed nibh ornare interdum eu eu mi."));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDocumentNotDividedViaHorizontalRulesWhenIsDisabled() {
|
||||
MarkdownDocumentReaderConfig config = MarkdownDocumentReaderConfig.builder()
|
||||
.withHorizontalRuleCreateDocument(false)
|
||||
.build();
|
||||
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/horizontal-rules.md", config);
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(1);
|
||||
|
||||
Document documentsFirst = documents.get(0);
|
||||
assertThat(documentsFirst.getMetadata()).isEmpty();
|
||||
assertThat(documentsFirst.getContent()).startsWith("Lorem ipsum dolor sit amet, consectetur adipiscing elit")
|
||||
.endsWith("Phasellus eget tellus sed nibh ornare interdum eu eu mi.");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSimpleMarkdownDocumentWithHardAndSoftLineBreaks() {
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/simple.md");
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(1);
|
||||
|
||||
Document documentsFirst = documents.get(0);
|
||||
assertThat(documentsFirst.getMetadata()).isEmpty();
|
||||
assertThat(documentsFirst.getContent()).isEqualTo(
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida. Cras accumsan tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu elementum dignissim.Nullam nisi dui, egestas nec sem nec, interdum lobortis enim. Pellentesque odio orci, faucibus eu luctus nec, venenatis et magna. Vestibulum nec eros non felis fermentum posuere eget ac risus.Aenean eu leo eu nibh tristique posuere quis quis massa. Nullam lacinia luctus sem ut vehicula.");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCode() {
|
||||
MarkdownDocumentReaderConfig config = MarkdownDocumentReaderConfig.builder()
|
||||
.withHorizontalRuleCreateDocument(true)
|
||||
.build();
|
||||
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/code.md", config);
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).satisfiesExactly(document -> {
|
||||
assertThat(document.getMetadata()).isEqualTo(Map.of());
|
||||
assertThat(document.getContent()).isEqualTo("This is a Java sample application:");
|
||||
}, document -> {
|
||||
assertThat(document.getMetadata()).isEqualTo(Map.of("lang", "java", "category", "code_block"));
|
||||
assertThat(document.getContent()).startsWith("package com.example.demo;")
|
||||
.contains("SpringApplication.run(DemoApplication.class, args);");
|
||||
}, document -> {
|
||||
assertThat(document.getMetadata()).isEqualTo(Map.of("category", "code_inline"));
|
||||
assertThat(document.getContent()).isEqualTo(
|
||||
"Markdown also provides the possibility to use inline code formatting throughout the entire sentence.");
|
||||
}, document -> {
|
||||
assertThat(document.getMetadata()).isEqualTo(Map.of());
|
||||
assertThat(document.getContent())
|
||||
.isEqualTo("Another possibility is to set block code without specific highlighting:");
|
||||
}, document -> {
|
||||
assertThat(document.getMetadata()).isEqualTo(Map.of("lang", "", "category", "code_block"));
|
||||
assertThat(document.getContent()).isEqualTo("./mvnw spring-javaformat:apply\n");
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCodeWhenCodeBlockShouldNotBeSeparatedDocument() {
|
||||
MarkdownDocumentReaderConfig config = MarkdownDocumentReaderConfig.builder()
|
||||
.withHorizontalRuleCreateDocument(true)
|
||||
.withIncludeCodeBlock(true)
|
||||
.build();
|
||||
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/code.md", config);
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).satisfiesExactly(document -> {
|
||||
assertThat(document.getMetadata()).isEqualTo(Map.of("lang", "java", "category", "code_block"));
|
||||
assertThat(document.getContent()).startsWith("This is a Java sample application: package com.example.demo")
|
||||
.contains("SpringApplication.run(DemoApplication.class, args);");
|
||||
}, document -> {
|
||||
assertThat(document.getMetadata()).isEqualTo(Map.of("category", "code_inline"));
|
||||
assertThat(document.getContent()).isEqualTo(
|
||||
"Markdown also provides the possibility to use inline code formatting throughout the entire sentence.");
|
||||
}, document -> {
|
||||
assertThat(document.getMetadata()).isEqualTo(Map.of("lang", "", "category", "code_block"));
|
||||
assertThat(document.getContent()).isEqualTo(
|
||||
"Another possibility is to set block code without specific highlighting: ./mvnw spring-javaformat:apply\n");
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBlockquote() {
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/blockquote.md");
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(2)
|
||||
.extracting(Document::getMetadata, Document::getContent)
|
||||
.containsOnly(tuple(Map.of(),
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue."),
|
||||
tuple(Map.of("category", "blockquote"),
|
||||
"Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit."));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBlockquoteWhenBlockquoteShouldNotBeSeparatedDocument() {
|
||||
MarkdownDocumentReaderConfig config = MarkdownDocumentReaderConfig.builder()
|
||||
.withIncludeBlockquote(true)
|
||||
.build();
|
||||
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/blockquote.md", config);
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(1);
|
||||
|
||||
Document documentsFirst = documents.get(0);
|
||||
assertThat(documentsFirst.getMetadata()).isEqualTo(Map.of("category", "blockquote"));
|
||||
assertThat(documentsFirst.getContent()).isEqualTo(
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue. Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit.");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLists() {
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/lists.md");
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(2)
|
||||
.extracting(Document::getMetadata, Document::getContent)
|
||||
.containsOnly(tuple(Map.of("category", "header_2", "title", "Ordered list"),
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue. Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien odio. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a porttitor."),
|
||||
tuple(Map.of("category", "header_2", "title", "Unordered list"),
|
||||
"Aenean eu leo eu nibh tristique posuere quis quis massa. Aenean imperdiet libero dui, nec malesuada dui maximus vel. Vestibulum sed dui condimentum, cursus libero in, dapibus tortor. Etiam facilisis enim in egestas dictum."));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWithAdditionalMetadata() {
|
||||
MarkdownDocumentReaderConfig config = MarkdownDocumentReaderConfig.builder()
|
||||
.withAdditionalMetadata("service", "some-service-name")
|
||||
.withAdditionalMetadata("env", "prod")
|
||||
.build();
|
||||
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader("classpath:/simple.md", config);
|
||||
|
||||
List<Document> documents = reader.get();
|
||||
|
||||
assertThat(documents).hasSize(1);
|
||||
|
||||
Document documentsFirst = documents.get(0);
|
||||
assertThat(documentsFirst.getMetadata()).isEqualTo(Map.of("service", "some-service-name", "env", "prod"));
|
||||
assertThat(documentsFirst.getContent()).startsWith("Lorem ipsum dolor sit amet, consectetur adipiscing elit.");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed
|
||||
nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue.
|
||||
|
||||
> Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget
|
||||
> sapien odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a
|
||||
> porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum
|
||||
> suscipit.
|
||||
|
||||
25
document-readers/markdown-reader/src/test/resources/code.md
Normal file
25
document-readers/markdown-reader/src/test/resources/code.md
Normal file
@@ -0,0 +1,25 @@
|
||||
This is a Java sample application:
|
||||
|
||||
```java
|
||||
package com.example.demo;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class DemoApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(DemoApplication.class, args);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Markdown also provides the possibility to `use inline code formatting throughout` the entire sentence.
|
||||
|
||||
---
|
||||
|
||||
Another possibility is to set block code without specific highlighting:
|
||||
|
||||
```
|
||||
./mvnw spring-javaformat:apply
|
||||
```
|
||||
@@ -0,0 +1,27 @@
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida.
|
||||
|
||||
---
|
||||
|
||||
Cras accumsan tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu
|
||||
elementum dignissim.
|
||||
|
||||
***
|
||||
Nullam nisi dui, egestas nec sem nec, interdum lobortis enim. Pellentesque odio orci, faucibus eu luctus nec, venenatis
|
||||
et magna.
|
||||
|
||||
* * *
|
||||
|
||||
Vestibulum nec eros non felis fermentum posuere eget ac risus. Curabitur et fringilla massa. Cras facilisis nec nisl sit
|
||||
amet sagittis.
|
||||
|
||||
*****
|
||||
|
||||
Aenean eu leo eu nibh tristique posuere quis quis massa. Nullam lacinia luctus sem ut vehicula.
|
||||
|
||||
---------------------------------------
|
||||
|
||||
Aenean quis vulputate mi. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Nam tincidunt nunc a tortor tincidunt, nec lobortis diam rhoncus.
|
||||
|
||||
- - -
|
||||
|
||||
Nulla facilisi. Phasellus eget tellus sed nibh ornare interdum eu eu mi.
|
||||
17
document-readers/markdown-reader/src/test/resources/lists.md
Normal file
17
document-readers/markdown-reader/src/test/resources/lists.md
Normal file
@@ -0,0 +1,17 @@
|
||||
## Ordered list
|
||||
|
||||
1. Lorem ipsum dolor sit *amet*, consectetur adipiscing elit. **Curabitur** diam eros, laoreet sit _amet_ cursus vitae,
|
||||
varius sed nisi.
|
||||
2. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue.
|
||||
3. Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget
|
||||
sapien odio.
|
||||
1. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum
|
||||
suscipit.
|
||||
2. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a porttitor.
|
||||
|
||||
## Unordered list
|
||||
|
||||
* Aenean eu leo eu nibh tristique posuere quis quis massa.
|
||||
* Aenean imperdiet libero dui, nec malesuada dui maximus vel. Vestibulum sed dui condimentum, cursus libero in, dapibus
|
||||
tortor.
|
||||
* Etiam facilisis enim in egestas dictum.
|
||||
@@ -0,0 +1,20 @@
|
||||
# Header 1a
|
||||
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed
|
||||
nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue.
|
||||
|
||||
# Header 1b
|
||||
|
||||
Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam lobortis risus libero, sed
|
||||
sollicitudin risus cursus in. Morbi enim metus, ornare vel lacinia eget, venenatis vel nibh.
|
||||
|
||||
## Header 2b
|
||||
|
||||
Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien
|
||||
odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero.
|
||||
|
||||
# Header 1c
|
||||
|
||||
## Header 2c
|
||||
|
||||
Ut rhoncus nec justo a porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit.
|
||||
@@ -0,0 +1,8 @@
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida. Cras accumsan
|
||||
tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu elementum dignissim.
|
||||
|
||||
Nullam nisi dui, egestas nec sem nec, interdum lobortis enim. Pellentesque odio orci, faucibus eu luctus nec, venenatis et magna. Vestibulum nec eros non felis fermentum posuere eget ac risus.
|
||||
|
||||
Aenean eu leo eu nibh tristique posuere quis quis massa.\
|
||||
Nullam lacinia luctus sem ut vehicula.
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# This is a fancy header name
|
||||
|
||||
Lorem ipsum dolor sit amet, **consectetur adipiscing elit**. Donec tincidunt velit non bibendum gravida. Cras accumsan
|
||||
tincidunt ornare. Donec hendrerit consequat tellus *blandit* accumsan. Aenean aliquam metus at ***arcu elementum***
|
||||
dignissim.
|
||||
|
||||
### Header 3
|
||||
|
||||
Aenean eu leo eu nibh tristique _posuere quis quis massa_.
|
||||
Reference in New Issue
Block a user