diff --git a/pom.xml b/pom.xml index d16bdc3..67106e4 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,12 @@ httpclient + + org.jsoup + jsoup + 1.15.3 + + org.projectlombok lombok diff --git a/readme.adoc b/readme.adoc index df8e23e..853b06b 100644 --- a/readme.adoc +++ b/readme.adoc @@ -183,3 +183,49 @@ Removing a line will omit that dependency upgrade. To distribute `ci/pipeline.properties` across all modules use: `$ infra distribute ci-properties $trainIteration` + +===== Broken Link Report + +Resolve external links in reference documentation and print their status. + +`$ docs check-links $trainIteration` + +|=== +|Flag |Description + +| --local +| read the documentation from disk (target module workspace directory) + +| --project +| only check links of a specific project (eg. `redis`) + +| --report +| only report errors of the given categories (ERROR,REDIRECT,OK). Default is ALL + +|=== + +.Examples - Check links of release train/module +[source,console] +---- +$ docs check-links Turing GA --report ERROR + +$ docs check-links Turing SR1 --local true --project redis +---- + +Resolve external links of any web page (remote/local) and print their status. + +`$ check-links $url` + +|=== +|Flag |Description + +| --report +| only report errors of the given categories (ERROR,REDIRECT,OK). Default is ALL + +|=== + +.Example - Check links on any url +[source,console] +---- +$ check-links file:///usr/git/spring-data-mongodb/target/site/reference/html/index.html --report ERROR,REDIRECT +---- diff --git a/src/main/java/org/springframework/data/release/build/BuildOperations.java b/src/main/java/org/springframework/data/release/build/BuildOperations.java index 27cdfa9..3365239 100644 --- a/src/main/java/org/springframework/data/release/build/BuildOperations.java +++ b/src/main/java/org/springframework/data/release/build/BuildOperations.java @@ -15,6 +15,8 @@ */ package org.springframework.data.release.build; +import static org.springframework.data.release.model.Projects.*; + import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -35,6 +37,7 @@ import org.springframework.data.release.model.Projects; import org.springframework.data.release.model.Train; import org.springframework.data.release.model.TrainIteration; import org.springframework.data.release.utils.Logger; +import org.springframework.data.util.Streamable; import org.springframework.plugin.core.PluginRegistry; import org.springframework.stereotype.Component; import org.springframework.util.Assert; @@ -146,6 +149,21 @@ public class BuildOperations { logger.log(iteration, "Build finished"); } + public void buildDocumentation(TrainIteration iteration) { + + executor.doWithBuildSystemOrdered(Streamable.of(iteration.getModulesExcept(BOM, COMMONS, BUILD)), BuildSystem::triggerDocumentationBuild); + + logger.log(iteration, "Documentation build finished"); + } + + public void buildDocumentation(ModuleIteration iteration) { + + // TODO: check if this call is fine + // executor.doWithBuildSystemOrdered(Streamable.of(iteration), BuildSystem::triggerDocumentationBuild); + + logger.log(iteration, "Documentation build finished"); + } + /** * Performs the release build for all modules in the given {@link TrainIteration}. * diff --git a/src/main/java/org/springframework/data/release/build/BuildSystem.java b/src/main/java/org/springframework/data/release/build/BuildSystem.java index f880a4d..61a1503 100644 --- a/src/main/java/org/springframework/data/release/build/BuildSystem.java +++ b/src/main/java/org/springframework/data/release/build/BuildSystem.java @@ -70,6 +70,8 @@ interface BuildSystem extends Plugin { M triggerBuild(M module); + M triggerDocumentationBuild(M module); + /** * Deploy artifacts for the given {@link ModuleIteration} using {@link DeploymentInformation}. * diff --git a/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java b/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java index 626b589..cf06232 100644 --- a/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java +++ b/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java @@ -355,6 +355,21 @@ class MavenBuildSystem implements BuildSystem { mvn.execute(module.getProject(), arguments); } + @Override + public M triggerDocumentationBuild(M module) { + + Project project = module.getProject(); + if(!isMavenProject(project)) { + logger.log(project, "Skipping project as no pom.xml could be found in the working directory!"); + return module; + } + + mvn.execute(project, CommandLine.of(Goal.CLEAN, Goal.INSTALL, SKIP_TESTS, profile("distribute"))); + + logger.log(project, "Successfully finished documentation build."); + return module; + } + /* * (non-Javadoc) * @see org.springframework.data.release.build.BuildSystem#triggerDistributionBuild(org.springframework.data.release.model.Module) diff --git a/src/main/java/org/springframework/data/release/documentation/DocumentationCommands.java b/src/main/java/org/springframework/data/release/documentation/DocumentationCommands.java new file mode 100644 index 0000000..958baa7 --- /dev/null +++ b/src/main/java/org/springframework/data/release/documentation/DocumentationCommands.java @@ -0,0 +1,114 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.release.documentation; + +import static org.springframework.data.release.model.Projects.*; + +import java.io.File; +import java.util.Arrays; +import java.util.concurrent.ExecutorService; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.springframework.data.release.CliComponent; +import org.springframework.data.release.TimedCommand; +import org.springframework.data.release.build.BuildOperations; +import org.springframework.data.release.cli.StaticResources; +import org.springframework.data.release.documentation.DocumentationOperations.ReportFlags; +import org.springframework.data.release.io.Workspace; +import org.springframework.data.release.model.ModuleIteration; +import org.springframework.data.release.model.Project; +import org.springframework.data.release.model.TrainIteration; +import org.springframework.data.release.utils.ExecutionUtils; +import org.springframework.data.release.utils.Logger; +import org.springframework.data.util.Streamable; +import org.springframework.shell.core.annotation.CliCommand; +import org.springframework.shell.core.annotation.CliOption; +import org.springframework.util.StringUtils; + +/** + * @author Christoph Strobl + */ +@CliComponent +@RequiredArgsConstructor +public class DocumentationCommands extends TimedCommand { + + private final @NonNull DocumentationOperations operations; + private final @NonNull ExecutorService executorService; + private final @NonNull BuildOperations buildOperations; + private final @NonNull Workspace workspace; + private final @NonNull Logger logger; + + @CliCommand("docs check-links") + public void checkLinks(@CliOption(key = "", mandatory = true) TrainIteration iteration, @CliOption(key = "project", mandatory = false) Project project, @CliOption(key = "local", mandatory = false, unspecifiedDefaultValue = "false") boolean preview, @CliOption(key = "report", mandatory = false) String options) { + + if (project != null) { + checkLinks(iteration.getModule(project), preview, options); + return; + } + + ExecutionUtils.run(executorService, Streamable.of(iteration.getModulesExcept(BUILD, BOM, COMMONS)), module -> { + checkLinks(module, preview, options); + }); + } + + @CliCommand("check-links") + public void checkLinks(@CliOption(key = "", mandatory = true) String url, @CliOption(key = "report", mandatory = false) String options) { + + String result = operations.checkDocumentation(url).prettyPrint(readFlags(options)); + System.out.printf("Link Statistic:\r\n%s", result); + } + + public void checkLinks(ModuleIteration module, boolean preview, String options) { + + String path; + if (preview) { + + buildOperations.buildDocumentation(module); + File projectDirectory = workspace.getProjectDirectory(module.getProject()); + if (!projectDirectory.exists()) { + logger.warn(module, "Unable to locate project directory"); + return; + } + File source = new File(projectDirectory, "target/site/reference/html/index.html"); + if (!source.exists()) { + logger.warn(module, "Unable to locate reference documentation html %", source); + return; + } + path = source.getPath(); + } else { + path = new StaticResources(module).getDocumentationUrl(); + } + + if(!StringUtils.hasText(path)) { + logger.warn(module, "Empty path for reference documentation."); + return; + } + + String result = operations.checkDocumentation(path).prettyPrint(readFlags(options)); + logger.log(module, "%s Documentation Link Statistic:\r\n%s", module, result); + } + + private static ReportFlags[] readFlags(String options) { + + ReportFlags[] flags = new ReportFlags[]{ReportFlags.ALL}; + if (options != null) { + flags = Arrays.stream(options.split(",")).map(ReportFlags::valueOf).toArray(ReportFlags[]::new); + } + return flags; + } +} + diff --git a/src/main/java/org/springframework/data/release/documentation/DocumentationOperations.java b/src/main/java/org/springframework/data/release/documentation/DocumentationOperations.java new file mode 100644 index 0000000..8d39e3b --- /dev/null +++ b/src/main/java/org/springframework/data/release/documentation/DocumentationOperations.java @@ -0,0 +1,202 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.release.documentation; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.fusesource.jansi.Ansi; +import org.fusesource.jansi.Ansi.Color; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.select.Elements; +import org.springframework.data.release.utils.Logger; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.util.ObjectUtils; + +/** + * @author Christoph Strobl + */ +@Component +@RequiredArgsConstructor +class DocumentationOperations { + + private final @NonNull Logger logger; + + PageStats checkDocumentation(String url) { + PageStats stats = new PageStats(); + stats.links(new LinkChecker(logger).inspect(url)); + return stats; + } + + enum ReportFlags { + + ERROR(1), REDIRECT(2), OK(4), ALL(8); + + int bin; + + ReportFlags(int bin) { + this.bin = bin; + } + + static int flagsOf(ReportFlags... flags) { + int value = 0; + for (ReportFlags opt : flags) { + value = value | opt.bin; + } + return value; + } + } + + static class PageStats { + + public static final int ERROR = 1; // Binary 00001 + public static final int REDIRECT = 2; // Binary 00010 + public static final int OK = 4; // Binary 00100 + public static final int ALL = 8; // Binary 01000 + + LinkStats linkStats; + + String prettyPrint(ReportFlags... options) { + + if (linkStats != null) { + return linkStats.prettyPrint(options); + } + return "PageStats n/a"; + } + + void links(LinkStats linkStats) { + this.linkStats = linkStats; + } + } + + + static class LinkStats { + + Map resultMap = new LinkedHashMap<>(200); + + public HttpStatus computeIfAbsent(String key, Function mappingFunction) { + return resultMap.computeIfAbsent(key, mappingFunction); + } + + int size() { + return resultMap.size(); + } + + String prettyPrint() { + return prettyPrint(ReportFlags.ALL); + } + + String prettyPrint(ReportFlags... options) { + + int flags = ObjectUtils.isEmpty(options) ? ReportFlags.flagsOf(ReportFlags.ALL) : ReportFlags.flagsOf(options); + + return resultMap.entrySet().stream().filter(entry -> { + if ((flags & ReportFlags.ALL.bin) == ReportFlags.ALL.bin) { + return true; + } + if (entry.getValue().is2xxSuccessful() && (flags & ReportFlags.OK.bin) == ReportFlags.OK.bin) { + return true; + } + if (entry.getValue().is3xxRedirection() && (flags & ReportFlags.REDIRECT.bin) == ReportFlags.REDIRECT.bin) { + return true; + } + if (entry.getValue().is4xxClientError() && (flags & ReportFlags.ERROR.bin) == ReportFlags.ERROR.bin) { + return true; + } + return false; + }).sorted(Comparator.comparingInt(o -> o.getValue().value())).map(entry -> { + Ansi ansi = Ansi.ansi(); + if (entry.getValue().is2xxSuccessful()) { + ansi.fg(Color.GREEN); + } else if (entry.getValue().is4xxClientError()) + ansi.fg(Color.RED); + else if (entry.getValue().is3xxRedirection()) { + ansi.fg(Color.YELLOW); + } + return ansi.a(entry.getValue()).fg(Color.DEFAULT).a(": " + entry.getKey()).toString(); + }).collect(Collectors.joining("\r\n")); + } + } + + + @RequiredArgsConstructor + static class LinkChecker { + + private final Logger logger; + + LinkStats inspect(String url) throws RuntimeException { + + logger.log("", "Collecting links from: %s", url); + + Document doc = null; + try { + if (url.startsWith("http://") || url.startsWith("https://")) { + doc = Jsoup.parse(new URL(url), 5000); + } else { + if (url.startsWith("file://")) { + url = url.replace("file://", ""); + } + doc = Jsoup.parse(new File(url), "UTF-8"); + } + } catch (IOException e) { + throw new IllegalStateException(e); + } + + Elements links = doc.select("a[href]"); // a with href + + logger.log("", "Found %s links.", links.size()); + + LinkStats stats = new LinkStats(); + + links.forEach(link -> { + if (link.attr("href").startsWith("#")) { + return; + } + checkUrl(link.attr("href"), stats); + }); + + logger.log("", "Analyzed %s external links.", stats.size()); + + return stats; + } + + private void checkUrl(String url, LinkStats stats) { + + stats.computeIfAbsent(url, key -> { + try { + HttpURLConnection connection = (HttpURLConnection) new URL(key).openConnection(); + connection.setRequestMethod("HEAD"); + connection.setConnectTimeout(5000); + connection.setReadTimeout(8000); + return HttpStatus.valueOf(connection.getResponseCode()); + } catch (Exception ex) { + return HttpStatus.valueOf(500); + } + }); + } + } +}