Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce checkstyle plugin

Browse Source 
- Based on https://github.com/spring-io/spring-javaformat
- In this iteration, checkstyles are only enabled for spring-ai-core
This commit is contained in:
Soby Chacko
2024-10-24 10:39:48 -04:00
committed by Mark Pollack
parent 33a72417e1
commit 8e758dbd00
1412 changed files with 26997 additions and 21963 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
document-readers/markdown-reader/src/main/java/org/springframework/ai/reader/markdown/MarkdownDocumentReader.java
View File

@@ -1,18 +1,45 @@
package org.springframework.ai.reader.markdown;
/*
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.commonmark.node.*;
import org.commonmark.parser.Parser;
import org.springframework.ai.document.Document;
import org.springframework.ai.document.DocumentReader;
import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
package org.springframework.ai.reader.markdown;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import org.commonmark.node.AbstractVisitor;
import org.commonmark.node.BlockQuote;
import org.commonmark.node.Code;
import org.commonmark.node.FencedCodeBlock;
import org.commonmark.node.HardLineBreak;
import org.commonmark.node.Heading;
import org.commonmark.node.ListItem;
import org.commonmark.node.Node;
import org.commonmark.node.SoftLineBreak;
import org.commonmark.node.Text;
import org.commonmark.node.ThematicBreak;
import org.commonmark.parser.Parser;
import org.springframework.ai.document.Document;
import org.springframework.ai.document.DocumentReader;
import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
/**
* Reads the given Markdown resource and groups headers, paragraphs, or text divided by
* horizontal lines (depending on the
@@ -58,10 +85,10 @@ public class MarkdownDocumentReader implements DocumentReader {
*/
@Override
public List<Document> get() {
try (var input = markdownResource.getInputStream()) {
Node node = parser.parseReader(new InputStreamReader(input));
try (var input = this.markdownResource.getInputStream()) {
Node node = this.parser.parseReader(new InputStreamReader(input));
DocumentVisitor documentVisitor = new DocumentVisitor(config);
DocumentVisitor documentVisitor = new DocumentVisitor(this.config);
node.accept(documentVisitor);
return documentVisitor.getDocuments();
@@ -90,7 +117,7 @@ public class MarkdownDocumentReader implements DocumentReader {
@Override
public void visit(org.commonmark.node.Document document) {
currentDocumentBuilder = Document.builder();
this.currentDocumentBuilder = Document.builder();
super.visit(document);
}
@@ -102,7 +129,7 @@ public class MarkdownDocumentReader implements DocumentReader {
@Override
public void visit(ThematicBreak thematicBreak) {
if (config.horizontalRuleCreateDocument) {
if (this.config.horizontalRuleCreateDocument) {
buildAndFlush();
}
super.visit(thematicBreak);
@@ -128,32 +155,32 @@ public class MarkdownDocumentReader implements DocumentReader {
@Override
public void visit(BlockQuote blockQuote) {
if (!config.includeBlockquote) {
if (!this.config.includeBlockquote) {
buildAndFlush();
}
translateLineBreakToSpace();
currentDocumentBuilder.withMetadata("category", "blockquote");
this.currentDocumentBuilder.withMetadata("category", "blockquote");
super.visit(blockQuote);
}
@Override
public void visit(Code code) {
currentParagraphs.add(code.getLiteral());
currentDocumentBuilder.withMetadata("category", "code_inline");
this.currentParagraphs.add(code.getLiteral());
this.currentDocumentBuilder.withMetadata("category", "code_inline");
super.visit(code);
}
@Override
public void visit(FencedCodeBlock fencedCodeBlock) {
if (!config.includeCodeBlock) {
if (!this.config.includeCodeBlock) {
buildAndFlush();
}
translateLineBreakToSpace();
currentParagraphs.add(fencedCodeBlock.getLiteral());
currentDocumentBuilder.withMetadata("category", "code_block");
currentDocumentBuilder.withMetadata("lang", fencedCodeBlock.getInfo());
this.currentParagraphs.add(fencedCodeBlock.getLiteral());
this.currentDocumentBuilder.withMetadata("category", "code_block");
this.currentDocumentBuilder.withMetadata("lang", fencedCodeBlock.getInfo());
buildAndFlush();
@@ -163,11 +190,11 @@ public class MarkdownDocumentReader implements DocumentReader {
@Override
public void visit(Text text) {
if (text.getParent() instanceof Heading heading) {
currentDocumentBuilder.withMetadata("category", "header_%d".formatted(heading.getLevel()))
this.currentDocumentBuilder.withMetadata("category", "header_%d".formatted(heading.getLevel()))
.withMetadata("title", text.getLiteral());
}
else {
currentParagraphs.add(text.getLiteral());
this.currentParagraphs.add(text.getLiteral());
}
super.visit(text);
@@ -176,29 +203,29 @@ public class MarkdownDocumentReader implements DocumentReader {
public List<Document> getDocuments() {
buildAndFlush();
return documents;
return this.documents;
}
private void buildAndFlush() {
if (!currentParagraphs.isEmpty()) {
String content = String.join("", currentParagraphs);
if (!this.currentParagraphs.isEmpty()) {
String content = String.join("", this.currentParagraphs);
Document.Builder builder = currentDocumentBuilder.withContent(content);
Document.Builder builder = this.currentDocumentBuilder.withContent(content);
config.additionalMetadata.forEach(builder::withMetadata);
this.config.additionalMetadata.forEach(builder::withMetadata);
Document document = builder.build();
documents.add(document);
this.documents.add(document);
currentParagraphs.clear();
this.currentParagraphs.clear();
}
currentDocumentBuilder = Document.builder();
this.currentDocumentBuilder = Document.builder();
}
private void translateLineBreakToSpace() {
if (!currentParagraphs.isEmpty()) {
currentParagraphs.add(" ");
if (!this.currentParagraphs.isEmpty()) {
this.currentParagraphs.add(" ");
}
}

30
document-readers/markdown-reader/src/main/java/org/springframework/ai/reader/markdown/config/MarkdownDocumentReaderConfig.java
View File

@@ -1,12 +1,28 @@
/*
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.reader.markdown.config;
import java.util.HashMap;
import java.util.Map;
import org.springframework.ai.document.Document;
import org.springframework.ai.reader.markdown.MarkdownDocumentReader;
import org.springframework.util.Assert;
import java.util.HashMap;
import java.util.Map;
/**
* Common configuration for the {@link MarkdownDocumentReader}.
*
@@ -23,10 +39,10 @@ public class MarkdownDocumentReaderConfig {
public final Map<String, Object> additionalMetadata;
public MarkdownDocumentReaderConfig(Builder builder) {
horizontalRuleCreateDocument = builder.horizontalRuleCreateDocument;
includeCodeBlock = builder.includeCodeBlock;
includeBlockquote = builder.includeBlockquote;
additionalMetadata = builder.additionalMetadata;
this.horizontalRuleCreateDocument = builder.horizontalRuleCreateDocument;
this.includeCodeBlock = builder.includeCodeBlock;
this.includeBlockquote = builder.includeBlockquote;
this.additionalMetadata = builder.additionalMetadata;
}
/**

25
document-readers/markdown-reader/src/test/java/org/springframework/ai/reader/markdown/MarkdownDocumentReaderTest.java
View File

@@ -1,12 +1,29 @@
package org.springframework.ai.reader.markdown;
/*
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.junit.jupiter.api.Test;
import org.springframework.ai.document.Document;
import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
package org.springframework.ai.reader.markdown;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.ai.document.Document;
import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.groups.Tuple.tuple;