Add command that reports broken links in reference documentation.

Does not check anchors.

$ docs check-links Turing GA
Collecting links from: https://docs.spring.io/spring-data/jpa/docs/3.0.0/reference/html/
Found 355 links.
Processing ..................................................
200 OK: https://github.com/spring-projects/spring-data-jpa
200 OK: https://github.com/spring-projects/spring-data-jpa/issues
200 OK: https://repo.spring.io/libs-release

Closes #22

pom.xml

@@ -84,6 +84,12 @@
 			<artifactId>httpclient</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>org.jsoup</groupId>
+			<artifactId>jsoup</artifactId>
+			<version>1.15.3</version>
+		</dependency>
+
 		<dependency>
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>

readme.adoc

@@ -183,3 +183,49 @@ Removing a line will omit that dependency upgrade.
 To distribute `ci/pipeline.properties` across all modules use:
 
 `$ infra distribute ci-properties $trainIteration`
+
+===== Broken Link Report
+
+Resolve external links in reference documentation and print their status.
+
+`$ docs check-links $trainIteration`
+
+|===
+|Flag |Description
+
+| --local
+| read the documentation from disk (target module workspace directory)
+
+| --project
+| only check links of a specific project (eg. `redis`)
+
+| --report
+| only report errors of the given categories (ERROR,REDIRECT,OK). Default is ALL
+
+|===
+
+.Examples - Check links of release train/module
+[source,console]
+----
+$ docs check-links Turing GA --report ERROR
+
+$ docs check-links Turing SR1 --local true --project redis
+----
+
+Resolve external links of any web page (remote/local) and print their status.
+
+`$ check-links $url`
+
+|===
+|Flag |Description
+
+| --report
+| only report errors of the given categories (ERROR,REDIRECT,OK). Default is ALL
+
+|===
+
+.Example - Check links on any url
+[source,console]
+----
+$ check-links file:///usr/git/spring-data-mongodb/target/site/reference/html/index.html --report ERROR,REDIRECT
+----

src/main/java/org/springframework/data/release/build/BuildOperations.java

@@ -15,6 +15,8 @@
  */
 package org.springframework.data.release.build;
 
+import static org.springframework.data.release.model.Projects.*;
+
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 
@@ -35,6 +37,7 @@ import org.springframework.data.release.model.Projects;
 import org.springframework.data.release.model.Train;
 import org.springframework.data.release.model.TrainIteration;
 import org.springframework.data.release.utils.Logger;
+import org.springframework.data.util.Streamable;
 import org.springframework.plugin.core.PluginRegistry;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
@@ -146,6 +149,21 @@ public class BuildOperations {
 		logger.log(iteration, "Build finished");
 	}
 
+	public void buildDocumentation(TrainIteration iteration) {
+
+		executor.doWithBuildSystemOrdered(Streamable.of(iteration.getModulesExcept(BOM, COMMONS, BUILD)), BuildSystem::triggerDocumentationBuild);
+
+		logger.log(iteration, "Documentation build finished");
+	}
+
+	public void buildDocumentation(ModuleIteration iteration) {
+
+		// TODO: check if this call is fine
+		// executor.doWithBuildSystemOrdered(Streamable.of(iteration), BuildSystem::triggerDocumentationBuild);
+
+		logger.log(iteration, "Documentation build finished");
+	}
+
 	/**
 	 * Performs the release build for all modules in the given {@link TrainIteration}.
 	 *

src/main/java/org/springframework/data/release/build/BuildSystem.java

@@ -70,6 +70,8 @@ interface BuildSystem extends Plugin<Project> {
 
 	<M extends ProjectAware> M triggerBuild(M module);
 
+	<M extends ProjectAware> M triggerDocumentationBuild(M module);
+
 	/**
 	 * Deploy artifacts for the given {@link ModuleIteration} using {@link DeploymentInformation}.
 	 *

src/main/java/org/springframework/data/release/build/MavenBuildSystem.java

@@ -355,6 +355,21 @@ class MavenBuildSystem implements BuildSystem {
 		mvn.execute(module.getProject(), arguments);
 	}
 
+	@Override
+	public <M extends ProjectAware> M triggerDocumentationBuild(M module) {
+
+		Project project = module.getProject();
+		if(!isMavenProject(project)) {
+			logger.log(project, "Skipping project as no pom.xml could be found in the working directory!");
+			return module;
+		}
+
+		mvn.execute(project, CommandLine.of(Goal.CLEAN, Goal.INSTALL, SKIP_TESTS, profile("distribute")));
+
+		logger.log(project, "Successfully finished documentation build.");
+		return module;
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see org.springframework.data.release.build.BuildSystem#triggerDistributionBuild(org.springframework.data.release.model.Module)

src/main/java/org/springframework/data/release/documentation/DocumentationCommands.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.release.documentation;
+
+import static org.springframework.data.release.model.Projects.*;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import org.springframework.data.release.CliComponent;
+import org.springframework.data.release.TimedCommand;
+import org.springframework.data.release.build.BuildOperations;
+import org.springframework.data.release.cli.StaticResources;
+import org.springframework.data.release.documentation.DocumentationOperations.ReportFlags;
+import org.springframework.data.release.io.Workspace;
+import org.springframework.data.release.model.ModuleIteration;
+import org.springframework.data.release.model.Project;
+import org.springframework.data.release.model.TrainIteration;
+import org.springframework.data.release.utils.ExecutionUtils;
+import org.springframework.data.release.utils.Logger;
+import org.springframework.data.util.Streamable;
+import org.springframework.shell.core.annotation.CliCommand;
+import org.springframework.shell.core.annotation.CliOption;
+import org.springframework.util.StringUtils;
+
+/**
+ * @author Christoph Strobl
+ */
+@CliComponent
+@RequiredArgsConstructor
+public class DocumentationCommands extends TimedCommand {
+
+	private final @NonNull DocumentationOperations operations;
+	private final @NonNull ExecutorService executorService;
+	private final @NonNull BuildOperations buildOperations;
+	private final @NonNull Workspace workspace;
+	private final @NonNull Logger logger;
+
+	@CliCommand("docs check-links")
+	public void checkLinks(@CliOption(key = "", mandatory = true) TrainIteration iteration, @CliOption(key = "project", mandatory = false) Project project, @CliOption(key = "local", mandatory = false, unspecifiedDefaultValue = "false") boolean preview, @CliOption(key = "report", mandatory = false) String options) {
+
+		if (project != null) {
+			checkLinks(iteration.getModule(project), preview, options);
+			return;
+		}
+
+		ExecutionUtils.run(executorService, Streamable.of(iteration.getModulesExcept(BUILD, BOM, COMMONS)), module -> {
+			checkLinks(module, preview, options);
+		});
+	}
+
+	@CliCommand("check-links")
+	public void checkLinks(@CliOption(key = "", mandatory = true) String url, @CliOption(key = "report", mandatory = false) String options) {
+
+		String result = operations.checkDocumentation(url).prettyPrint(readFlags(options));
+		System.out.printf("Link Statistic:\r\n%s", result);
+	}
+
+	public void checkLinks(ModuleIteration module, boolean preview, String options) {
+
+		String path;
+		if (preview) {
+
+			buildOperations.buildDocumentation(module);
+			File projectDirectory = workspace.getProjectDirectory(module.getProject());
+			if (!projectDirectory.exists()) {
+				logger.warn(module, "Unable to locate project directory");
+				return;
+			}
+			File source = new File(projectDirectory, "target/site/reference/html/index.html");
+			if (!source.exists()) {
+				logger.warn(module, "Unable to locate reference documentation html %", source);
+				return;
+			}
+			path = source.getPath();
+		} else {
+			path = new StaticResources(module).getDocumentationUrl();
+		}
+
+		if(!StringUtils.hasText(path)) {
+			logger.warn(module, "Empty path for reference documentation.");
+			return;
+		}
+
+		String result = operations.checkDocumentation(path).prettyPrint(readFlags(options));
+		logger.log(module, "%s Documentation Link Statistic:\r\n%s", module, result);
+	}
+
+	private static ReportFlags[] readFlags(String options) {
+
+		ReportFlags[] flags = new ReportFlags[]{ReportFlags.ALL};
+		if (options != null) {
+			flags = Arrays.stream(options.split(",")).map(ReportFlags::valueOf).toArray(ReportFlags[]::new);
+		}
+		return flags;
+	}
+}
+

src/main/java/org/springframework/data/release/documentation/DocumentationOperations.java

@@ -0,0 +1,202 @@
+/*
+ * Copyright 2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.release.documentation;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import org.fusesource.jansi.Ansi;
+import org.fusesource.jansi.Ansi.Color;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+import org.springframework.data.release.utils.Logger;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Component;
+import org.springframework.util.ObjectUtils;
+
+/**
+ * @author Christoph Strobl
+ */
+@Component
+@RequiredArgsConstructor
+class DocumentationOperations {
+
+	private final @NonNull Logger logger;
+
+	PageStats checkDocumentation(String url) {
+		PageStats stats = new PageStats();
+		stats.links(new LinkChecker(logger).inspect(url));
+		return stats;
+	}
+
+	enum ReportFlags {
+
+		ERROR(1), REDIRECT(2), OK(4), ALL(8);
+
+		int bin;
+
+		ReportFlags(int bin) {
+			this.bin = bin;
+		}
+
+		static int flagsOf(ReportFlags... flags) {
+			int value = 0;
+			for (ReportFlags opt : flags) {
+				value = value | opt.bin;
+			}
+			return value;
+		}
+	}
+
+	static class PageStats {
+
+		public static final int ERROR = 1;  // Binary 00001
+		public static final int REDIRECT = 2;  // Binary 00010
+		public static final int OK = 4;  // Binary 00100
+		public static final int ALL = 8;  // Binary 01000
+
+		LinkStats linkStats;
+
+		String prettyPrint(ReportFlags... options) {
+
+			if (linkStats != null) {
+				return linkStats.prettyPrint(options);
+			}
+			return "PageStats n/a";
+		}
+
+		void links(LinkStats linkStats) {
+			this.linkStats = linkStats;
+		}
+	}
+
+
+	static class LinkStats {
+
+		Map<String, HttpStatus> resultMap = new LinkedHashMap<>(200);
+
+		public HttpStatus computeIfAbsent(String key, Function<? super String, ? extends HttpStatus> mappingFunction) {
+			return resultMap.computeIfAbsent(key, mappingFunction);
+		}
+
+		int size() {
+			return resultMap.size();
+		}
+
+		String prettyPrint() {
+			return prettyPrint(ReportFlags.ALL);
+		}
+
+		String prettyPrint(ReportFlags... options) {
+
+			int flags = ObjectUtils.isEmpty(options) ? ReportFlags.flagsOf(ReportFlags.ALL) : ReportFlags.flagsOf(options);
+
+			return resultMap.entrySet().stream().filter(entry -> {
+				if ((flags & ReportFlags.ALL.bin) == ReportFlags.ALL.bin) {
+					return true;
+				}
+				if (entry.getValue().is2xxSuccessful() && (flags & ReportFlags.OK.bin) == ReportFlags.OK.bin) {
+					return true;
+				}
+				if (entry.getValue().is3xxRedirection() && (flags & ReportFlags.REDIRECT.bin) == ReportFlags.REDIRECT.bin) {
+					return true;
+				}
+				if (entry.getValue().is4xxClientError() && (flags & ReportFlags.ERROR.bin) == ReportFlags.ERROR.bin) {
+					return true;
+				}
+				return false;
+			}).sorted(Comparator.comparingInt(o -> o.getValue().value())).map(entry -> {
+				Ansi ansi = Ansi.ansi();
+				if (entry.getValue().is2xxSuccessful()) {
+					ansi.fg(Color.GREEN);
+				} else if (entry.getValue().is4xxClientError())
+					ansi.fg(Color.RED);
+				else if (entry.getValue().is3xxRedirection()) {
+					ansi.fg(Color.YELLOW);
+				}
+				return ansi.a(entry.getValue()).fg(Color.DEFAULT).a(": " + entry.getKey()).toString();
+			}).collect(Collectors.joining("\r\n"));
+		}
+	}
+
+
+	@RequiredArgsConstructor
+	static class LinkChecker {
+
+		private final Logger logger;
+
+		LinkStats inspect(String url) throws RuntimeException {
+
+			logger.log("", "Collecting links from: %s", url);
+
+			Document doc = null;
+			try {
+				if (url.startsWith("http://") || url.startsWith("https://")) {
+					doc = Jsoup.parse(new URL(url), 5000);
+				} else {
+					if (url.startsWith("file://")) {
+						url = url.replace("file://", "");
+					}
+					doc = Jsoup.parse(new File(url), "UTF-8");
+				}
+			} catch (IOException e) {
+				throw new IllegalStateException(e);
+			}
+
+			Elements links = doc.select("a[href]"); // a with href
+
+			logger.log("", "Found %s links.", links.size());
+
+			LinkStats stats = new LinkStats();
+
+			links.forEach(link -> {
+				if (link.attr("href").startsWith("#")) {
+					return;
+				}
+				checkUrl(link.attr("href"), stats);
+			});
+
+			logger.log("", "Analyzed %s external links.", stats.size());
+
+			return stats;
+		}
+
+		private void checkUrl(String url, LinkStats stats) {
+
+			stats.computeIfAbsent(url, key -> {
+				try {
+					HttpURLConnection connection = (HttpURLConnection) new URL(key).openConnection();
+					connection.setRequestMethod("HEAD");
+					connection.setConnectTimeout(5000);
+					connection.setReadTimeout(8000);
+					return HttpStatus.valueOf(connection.getResponseCode());
+				} catch (Exception ex) {
+					return HttpStatus.valueOf(500);
+				}
+			});
+		}
+	}
+}