From e1f220fe255e9651c30050c97cd48048b4994167 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Sun, 22 Nov 2020 15:12:42 +0100 Subject: [PATCH] #160 - Attach changelog to GitHub release. --- release-tools/readme.md | 2 + .../data/release/git/VersionTags.java | 13 +- .../issues/github/ChangelogGenerator.java | 138 ++++++++++++++++++ .../issues/github/ChangelogSection.java | 75 ++++++++++ .../issues/github/ChangelogSections.java | 81 ++++++++++ .../data/release/issues/github/GitHub.java | 124 +++++++++++++++- .../release/issues/github/GitHubCommands.java | 35 +++++ .../release/issues/github/GitHubIssue.java | 32 +++- .../release/issues/github/GitHubRelease.java | 40 +++++ .../release/issues/github/GitHubUser.java | 40 +++++ .../release/issues/github/PullRequest.java | 37 +++++ .../release/model/DocumentationMetadata.java | 81 ++++++++++ .../data/release/sagan/ProjectMetadata.java | 44 +----- 13 files changed, 688 insertions(+), 54 deletions(-) create mode 100644 release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogGenerator.java create mode 100644 release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogSection.java create mode 100644 release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogSections.java create mode 100644 release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubRelease.java create mode 100644 release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubUser.java create mode 100644 release-tools/src/main/java/org/springframework/data/release/issues/github/PullRequest.java create mode 100644 release-tools/src/main/java/org/springframework/data/release/model/DocumentationMetadata.java diff --git a/release-tools/readme.md b/release-tools/readme.md index b9dba2a..a6229aa 100644 --- a/release-tools/readme.md +++ b/release-tools/readme.md @@ -48,6 +48,8 @@ $ release build $trainIteration $ release conclude $trainIteration $ git push $trainIteration $ git push $trainIteration --tags +# After GitHub issues migration: +# github push $trainIteration $ git backport changelog $trainIteration --target $targets $ foreach $target -> git push $target ``` diff --git a/release-tools/src/main/java/org/springframework/data/release/git/VersionTags.java b/release-tools/src/main/java/org/springframework/data/release/git/VersionTags.java index 4a9d1d6..a4f62c2 100644 --- a/release-tools/src/main/java/org/springframework/data/release/git/VersionTags.java +++ b/release-tools/src/main/java/org/springframework/data/release/git/VersionTags.java @@ -18,6 +18,7 @@ package org.springframework.data.release.git; import lombok.EqualsAndHashCode; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -69,13 +70,23 @@ public class VersionTags implements Streamable { sorted().collect(Collectors.toList()); } + /** + * Creates an empty {@link VersionTags} object for {@link Project}. + * + * @param project must not be {@literal null}. + * @return + */ + public static VersionTags empty(Project project) { + return new VersionTags(project, Collections.emptyList()); + } + /** * Returns the latest {@link Tag}. * * @return */ public Tag getLatest() { - return tags.get(tags.size() - 1); + return tags.isEmpty() ? null : tags.get(tags.size() - 1); } public Tag createTag(ModuleIteration iteration) { diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogGenerator.java b/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogGenerator.java new file mode 100644 index 0000000..39f7241 --- /dev/null +++ b/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogGenerator.java @@ -0,0 +1,138 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.data.release.issues.github; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Component; + +/** + * Generates a changelog markdown file which includes bug fixes, enhancements and contributors for a given milestone. + * + * @author Madhura Bhave + * @author Phillip Webb + */ +@Component +public class ChangelogGenerator { + + private static final Pattern ghUserMentionPattern = Pattern.compile("(^|[^\\w`])(@[\\w-]+)"); + + private final Set excludeLabels; + + private final Set excludeContributors; + + private final String contributorsTitle; + + private final ChangelogSections sections; + + public ChangelogGenerator() { + this.excludeLabels = new HashSet<>(Arrays.asList("type: task")); + this.excludeContributors = Collections.emptySet(); + this.contributorsTitle = null; + this.sections = new ChangelogSections(); + } + + /** + * Generates a file at the given path which includes bug fixes, enhancements and contributors for the given milestone. + * + * @param issues the issues to generate the changelog for + * @param sectionContentPostProcessor the postprocessor for a changelog section + */ + public String generate(List issues, + BiFunction sectionContentPostProcessor) { + return generateContent(issues, sectionContentPostProcessor); + } + + private boolean isExcluded(GitHubIssue issue) { + return issue.getLabels().stream().anyMatch(this::isExcluded); + } + + private boolean isExcluded(Label label) { + return this.excludeLabels.contains(label.getName()); + } + + private String generateContent(List issues, + BiFunction sectionContentPostProcessor) { + StringBuilder content = new StringBuilder(); + addSectionContent(content, this.sections.collate(issues), sectionContentPostProcessor); + Set contributors = getContributors(issues); + if (!contributors.isEmpty()) { + addContributorsContent(content, contributors); + } + return content.toString(); + } + + private void addSectionContent(StringBuilder result, Map> sectionIssues, + BiFunction sectionContentPostProcessor) { + + sectionIssues.forEach((section, issues) -> { + + issues.sort(Comparator.reverseOrder()); + + StringBuilder content = new StringBuilder(); + + content.append((content.length() != 0) ? String.format("%n") : ""); + content.append("## ").append(section).append(String.format("%n%n")); + issues.stream().map(this::getFormattedIssue).forEach(content::append); + + result.append(sectionContentPostProcessor.apply(section, content.toString())); + }); + } + + private String getFormattedIssue(GitHubIssue issue) { + String title = issue.getTitle(); + title = ghUserMentionPattern.matcher(title).replaceAll("$1`$2`"); + return String.format("- %s %s%n", title, getLinkToIssue(issue)); + } + + private String getLinkToIssue(GitHubIssue issue) { + return "[" + issue.getId() + "]" + "(" + issue.getUrl() + ")"; + } + + private Set getContributors(List issues) { + if (this.excludeContributors.contains("*")) { + return Collections.emptySet(); + } + return issues.stream().filter((issue) -> issue.getPullRequest() != null).map(GitHubIssue::getUser) + .filter(this::isIncludedContributor).collect(Collectors.toSet()); + } + + private boolean isIncludedContributor(GitHubUser user) { + return !this.excludeContributors.contains(user.getName()); + } + + private void addContributorsContent(StringBuilder content, Set contributors) { + content.append(String.format("%n## ")); + content.append((this.contributorsTitle != null) ? this.contributorsTitle : ":heart: Contributors"); + content.append(String.format("%n%nWe'd like to thank all the contributors who worked on this release!%n%n")); + contributors.stream().map(this::formatContributors).forEach(content::append); + } + + private String formatContributors(GitHubUser c) { + return String.format("- [@%s](%s)%n", c.getName(), c.getUrl()); + } + +} diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogSection.java b/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogSection.java new file mode 100644 index 0000000..f974821 --- /dev/null +++ b/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogSection.java @@ -0,0 +1,75 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.data.release.issues.github; + +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Set; + +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +/** + * A single section of a changelog report. + * + * @author Phillip Webb + */ +class ChangelogSection { + + private final String title; + + private final String group; + + private final Set labels; + + ChangelogSection(String title, String group, String... labels) { + this(title, group, new LinkedHashSet<>(Arrays.asList(labels))); + } + + ChangelogSection(String title, String group, Set labels) { + Assert.hasText(title, "Title must not be empty"); + Assert.isTrue(!CollectionUtils.isEmpty(labels), "Labels must not be empty"); + this.title = title; + this.group = group; + this.labels = labels; + } + + String getGroup() { + return this.group; + } + + boolean isMatchFor(GitHubIssue issue) { + for (String candidate : this.labels) { + for (Label label : issue.getLabels()) { + if (label.getName().contains(candidate)) { + return true; + } + } + } + return false; + } + + public boolean hasLabel(String label) { + return this.labels.contains(label); + } + + @Override + public String toString() { + return this.title; + } + +} diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogSections.java b/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogSections.java new file mode 100644 index 0000000..6ae2bc7 --- /dev/null +++ b/release-tools/src/main/java/org/springframework/data/release/issues/github/ChangelogSections.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.release.issues.github; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * Manages sections of the changelog report. + * + * @author Phillip Webb + */ +class ChangelogSections { + + private static final List DEFAULT_SECTIONS; + static { + List sections = new ArrayList<>(); + add(sections, ":star: New Features", "enhancement"); + add(sections, ":beetle: Bug Fixes", "bug", "regression"); + add(sections, ":notebook_with_decorative_cover: Documentation", "documentation"); + add(sections, ":hammer: Dependency Upgrades", "dependency-upgrade"); + DEFAULT_SECTIONS = Collections.unmodifiableList(sections); + } + + private static void add(List sections, String title, String... labels) { + sections.add(new ChangelogSection(title, null, labels)); + } + + private final List sections; + + ChangelogSections() { + this.sections = DEFAULT_SECTIONS; + } + + Map> collate(List issues) { + + SortedMap> collated = new TreeMap<>( + Comparator.comparing(this.sections::indexOf)); + + for (GitHubIssue issue : issues) { + List sections = getSections(issue); + for (ChangelogSection section : sections) { + collated.computeIfAbsent(section, (key) -> new ArrayList<>()); + collated.get(section).add(issue); + } + } + return collated; + } + + private List getSections(GitHubIssue issue) { + List result = new ArrayList<>(); + Set groupClaimes = new HashSet<>(); + for (ChangelogSection section : this.sections) { + if (section.isMatchFor(issue) && groupClaimes.add(section.getGroup())) { + result.add(section); + } + } + return result; + } + +} diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHub.java b/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHub.java index a65df24..9ac8087 100644 --- a/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHub.java +++ b/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHub.java @@ -34,12 +34,17 @@ import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.core.ParameterizedTypeReference; import org.springframework.data.release.git.GitProject; +import org.springframework.data.release.git.Tag; +import org.springframework.data.release.git.VersionTags; import org.springframework.data.release.issues.Changelog; import org.springframework.data.release.issues.IssueTracker; import org.springframework.data.release.issues.Ticket; import org.springframework.data.release.issues.TicketReference; import org.springframework.data.release.issues.Tickets; import org.springframework.data.release.issues.github.GitHubIssue.Milestone; +import org.springframework.data.release.model.ArtifactVersion; +import org.springframework.data.release.model.DocumentationMetadata; +import org.springframework.data.release.model.Iteration; import org.springframework.data.release.model.ModuleIteration; import org.springframework.data.release.model.Project; import org.springframework.data.release.model.Tracker; @@ -69,6 +74,9 @@ class GitHub extends GitHubSupport implements IssueTracker { private static final String MILESTONE_BY_ID_URI_TEMPLATE = "/repos/spring-projects/{repoName}/milestones/{id}"; private static final String ISSUE_BY_ID_URI_TEMPLATE = "/repos/spring-projects/{repoName}/issues/{id}"; private static final String ISSUES_URI_TEMPLATE = "/repos/spring-projects/{repoName}/issues"; + private static final String RELEASE_BY_TAG_URI_TEMPLATE = "/repos/spring-projects/{repoName}/releases/tags/{tag}"; + private static final String RELEASE_URI_TEMPLATE = "/repos/spring-projects/{repoName}/releases"; + private static final String RELEASE_BY_ID_URI_TEMPLATE = "/repos/spring-projects/{repoName}/releases/{id}"; private static final ParameterizedTypeReference> MILESTONES_TYPE = new ParameterizedTypeReference>() {}; private static final ParameterizedTypeReference> ISSUES_TYPE = new ParameterizedTypeReference>() {}; @@ -436,6 +444,12 @@ class GitHub extends GitHubSupport implements IssueTracker { @Override public Tickets findTickets(ModuleIteration moduleIteration, List ticketReferences) { + return findGitHubIssues(moduleIteration, ticketReferences).stream().map(GitHub::toTicket) + .collect(Tickets.toTicketsCollector()); + } + + List findGitHubIssues(ModuleIteration moduleIteration, List ticketReferences) { + logger.log(moduleIteration, "Looking up GitHub issues from milestone …"); Map issues = getIssuesFor(moduleIteration, false, true) @@ -448,13 +462,14 @@ class GitHub extends GitHubSupport implements IssueTracker { Streamable.of(() -> ticketReferences.stream().filter(it -> it.getId().startsWith("#"))), ticketReference -> getTicket(issues, repositoryName, ticketReference)); - Tickets tickets = foundIssues.stream().map(GitHub::toTicket) - .filter(it -> it.isReleaseTicketFor(moduleIteration) || !it.isReleaseTicket()) - .collect(Tickets.toTicketsCollector()); + List gitHubIssues = foundIssues.stream().filter(it -> { + Ticket ticket = toTicket(it); + return !ticket.isReleaseTicketFor(moduleIteration) && !ticket.isReleaseTicket(); + }).collect(Collectors.toList()); - logger.log(moduleIteration, "Resolved %s tickets.", tickets.getOverallTotal()); + logger.log(moduleIteration, "Resolved %s tickets.", gitHubIssues.size()); - return tickets; + return gitHubIssues; } private GitHubIssue getTicket(Map cache, String repositoryName, TicketReference reference) { @@ -486,11 +501,9 @@ class GitHub extends GitHubSupport implements IssueTracker { try { - GitHubIssue gitHubIssue = operations.exchange(ISSUE_BY_ID_URI_TEMPLATE, HttpMethod.GET, + return operations.exchange(ISSUE_BY_ID_URI_TEMPLATE, HttpMethod.GET, new HttpEntity<>(new HttpHeaders()), ISSUE_TYPE, parameters).getBody(); - return gitHubIssue; - } catch (HttpStatusCodeException e) { if (e.getStatusCode() == HttpStatus.NOT_FOUND) { @@ -501,6 +514,100 @@ class GitHub extends GitHubSupport implements IssueTracker { } } + /** + * @param module + * @param ticketReferences + */ + public void createOrUpdateRelease(ModuleIteration module, List ticketReferences) { + + logger.log(module, "Preparing GitHub Release …"); + + List gitHubIssues = findGitHubIssues(module, ticketReferences); + + ArtifactVersion version = ArtifactVersion.of(module); + DocumentationMetadata documentation = DocumentationMetadata.of(module.getProject(), version); + + ChangelogGenerator generator = new ChangelogGenerator(); + String releaseBody = generator.generate(gitHubIssues, (changelogSection, s) -> s); + String documentationLinks = getDocumentationLinks(module, documentation); + + createOrUpdateRelease(module, String.format("## :green_book: Links%n%s%n%s%n", documentationLinks, releaseBody)); + logger.log(module, "GitHub Release up to date"); + } + + private String getDocumentationLinks(ModuleIteration module, DocumentationMetadata documentation) { + + String referenceDocUrl = documentation.getReferenceDocUrl(module.getTrain()); + String apiDocUrl = documentation.getApiDocUrl(module.getTrain()); + + String reference = String.format("* [%s %s Reference documentation](%s)", module.getProject().getFullName(), + documentation.getVersion(module.getTrain()), referenceDocUrl); + + String apidoc = String.format("* [%s %s Javadoc](%s)", module.getProject().getFullName(), + documentation.getVersion(module.getTrain()), apiDocUrl); + + return String.format("%s%n%s%n", reference, apidoc); + } + + private void createOrUpdateRelease(ModuleIteration module, String body) { + + String repositoryName = GitProject.of(module.getProject()).getRepositoryName(); + Tag tag = VersionTags.empty(module.getProject()).createTag(module); + logger.log(module, "Looking up GitHub Release …"); + + Iteration iteration = module.getTrainIteration().getIteration(); + boolean prerelase = iteration.isPreview(); + GitHubRelease release = findRelease(repositoryName, tag.getName()); + + if (release == null) { + release = new GitHubRelease(null, tag.getName(), tag.getName(), body, false, prerelase); + logger.log(module, "Creating new Release …"); + createRelease(repositoryName, release); + } else { + release = release.withPrerelease(prerelase).withBody(body); + logger.log(module, "Updating new Release …"); + updateRelease(repositoryName, release); + } + } + + private GitHubRelease findRelease(String repositoryName, String tagName) { + + Map parameters = newUrlTemplateVariables(); + parameters.put("repoName", repositoryName); + parameters.put("tag", tagName); + + try { + return operations.exchange(RELEASE_BY_TAG_URI_TEMPLATE, HttpMethod.GET, new HttpEntity<>(new HttpHeaders()), + GitHubRelease.class, parameters).getBody(); + } catch (HttpStatusCodeException e) { + + if (e.getStatusCode() == HttpStatus.NOT_FOUND) { + return null; + } + + throw e; + } + } + + private void createRelease(String repositoryName, GitHubRelease release) { + + Map parameters = newUrlTemplateVariables(); + parameters.put("repoName", repositoryName); + + operations.exchange(RELEASE_URI_TEMPLATE, HttpMethod.POST, new HttpEntity<>(release), GitHubRelease.class, + parameters); + } + + private void updateRelease(String repositoryName, GitHubRelease release) { + + Map parameters = newUrlTemplateVariables(); + parameters.put("repoName", repositoryName); + parameters.put("id", release.getId()); + + operations.exchange(RELEASE_BY_ID_URI_TEMPLATE, HttpMethod.PATCH, new HttpEntity<>(release), GitHubRelease.class, + parameters); + } + private Stream getIssuesFor(ModuleIteration moduleIteration, boolean forCurrentUser, boolean ignoreMissingMilestone) { @@ -556,4 +663,5 @@ class GitHub extends GitHubSupport implements IssueTracker { private static Ticket toTicket(GitHubIssue issue) { return new Ticket(issue.getId(), issue.getTitle(), issue.getHtmlUrl(), new GithubTicketStatus(issue.getState())); } + } diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubCommands.java b/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubCommands.java index c4981b2..4efc8ff 100644 --- a/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubCommands.java +++ b/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubCommands.java @@ -20,17 +20,25 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; +import java.util.List; import java.util.concurrent.Executor; import org.springframework.data.release.CliComponent; import org.springframework.data.release.TimedCommand; +import org.springframework.data.release.git.GitOperations; import org.springframework.data.release.issues.IssueTracker; +import org.springframework.data.release.issues.TicketReference; import org.springframework.data.release.model.Project; +import org.springframework.data.release.model.Tracker; +import org.springframework.data.release.model.TrainIteration; +import org.springframework.data.release.utils.ExecutionUtils; import org.springframework.plugin.core.PluginRegistry; import org.springframework.shell.core.annotation.CliCommand; import org.springframework.shell.core.annotation.CliOption; /** + * Component to execute GitHub related operations. + * * @author Mark Paluch */ @CliComponent @@ -39,6 +47,8 @@ import org.springframework.shell.core.annotation.CliOption; class GitHubCommands extends TimedCommand { @NonNull PluginRegistry tracker; + @NonNull GitHub gitHub; + @NonNull GitOperations git; @NonNull GitHubLabels gitHubLabels; @NonNull Executor executor; @@ -47,4 +57,29 @@ class GitHubCommands extends TimedCommand { gitHubLabels.createOrUpdateLabels(project); } + @CliCommand(value = "github push") + public void push(@CliOption(key = "", mandatory = true) TrainIteration iteration) { + + git.push(iteration); + git.pushTags(iteration.getTrain()); + + createOrUpdateLabels(iteration); + } + + @CliCommand(value = "github create release") + public void createOrUpdateLabels(@CliOption(key = "", mandatory = true) TrainIteration iteration) { + + TrainIteration previousIteration = git.getPreviousIteration(iteration); + + ExecutionUtils.run(executor, iteration, it -> { + + if (it.getProject().getTracker() == Tracker.GITHUB) { + + List ticketReferences = git.getTicketReferencesBetween(it.getProject(), previousIteration, + iteration); + gitHub.createOrUpdateRelease(it, ticketReferences); + } + }); + } + } diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubIssue.java b/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubIssue.java index ffc5d5e..2e08eb4 100644 --- a/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubIssue.java +++ b/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHubIssue.java @@ -31,41 +31,59 @@ import com.fasterxml.jackson.annotation.JsonProperty; * @author Mark Paluch */ @Value -class GitHubIssue { +class GitHubIssue implements Comparable { - String number, title, state; + String number, title, state, url; + GitHubUser user; List assignees; String htmlUrl; Object milestone; + PullRequest pullRequest; + List