Spring/spring-tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reconciling for github repo uris in concourse pipelines

Browse Source
This commit is contained in:
Kris De Volder
2017-12-27 14:45:03 -08:00
parent a3e4ce401a
commit f2e90f62ce
7 changed files with 329 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless-services/concourse-language-server/src/main/java/org/springframework/ide/vscode/concourse/PipelineYmlSchema.java
View File

@@ -49,6 +49,7 @@ import org.springframework.ide.vscode.concourse.ConcourseModel.ResourceModel;
import org.springframework.ide.vscode.concourse.ConcourseModel.StepModel;
import org.springframework.ide.vscode.concourse.github.GithubInfoProvider;
import org.springframework.ide.vscode.concourse.github.GithubRepoContentAssistant;
import org.springframework.ide.vscode.concourse.github.GithubValueParsers;
import org.yaml.snakeyaml.nodes.MappingNode;
import org.yaml.snakeyaml.nodes.Node;
@@ -424,6 +425,7 @@ public class PipelineYmlSchema implements YamlSchema {
{
AbstractType t_git_repo_uri = f.yatomic("GitRepoUri");
t_git_repo_uri.setCustomContentAssistant(new GithubRepoContentAssistant(github));
t_git_repo_uri.parseWith(GithubValueParsers.uri(github));
AbstractType source = f.ybean("GitSource");
addProp(source, "uri", t_git_repo_uri).isPrimary(true);

3
headless-services/concourse-language-server/src/main/java/org/springframework/ide/vscode/concourse/PipelineYmlSchemaProblems.java
View File

@@ -16,5 +16,6 @@ import org.springframework.ide.vscode.commons.languageserver.reconcile.ProblemTy
import static org.springframework.ide.vscode.commons.yaml.reconcile.YamlSchemaProblems.*;
public class PipelineYmlSchemaProblems {
protected static final ProblemType UNUSED_RESOURCE = problemType("PipelineYamlUnusedResource", ProblemSeverity.ERROR);
public static final ProblemType UNUSED_RESOURCE = problemType("PipelineYamlUnusedResource", ProblemSeverity.ERROR);
public static final ProblemType UNKNOWN_GITHUB_ENTITY = problemType("UnknownGithubEntitity", ProblemSeverity.WARNING);
}

73
headless-services/concourse-language-server/src/main/java/org/springframework/ide/vscode/concourse/github/DefaultGithubInfoProvider.java
View File

@@ -13,12 +13,15 @@ package org.springframework.ide.vscode.concourse.github;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import org.kohsuke.github.GHFileNotFoundException;
import org.kohsuke.github.GHPerson;
import org.kohsuke.github.GHRepository;
import org.kohsuke.github.GitHub;
import org.springframework.ide.vscode.commons.util.CollectorUtil;
import org.springframework.ide.vscode.commons.util.ExceptionUtil;
import org.springframework.ide.vscode.commons.util.Log;
import com.google.common.cache.Cache;
@@ -36,11 +39,39 @@ public class DefaultGithubInfoProvider implements GithubInfoProvider {
//especially when connecting fails. So that user may try to address the
//issue and try again.
private static class Result<T> {
private Object result;
public Result(Object valueOrTrowable) {
this.result = valueOrTrowable;
}
@SuppressWarnings("unchecked")
public T get() throws Exception {
if (result instanceof Throwable) {
throw ExceptionUtil.exception((Throwable) result);
}
return (T)result;
}
}
private <T> Callable<Result<T>> loader(Callable<T> callable) {
return () -> load(callable);
}
private static <T> Result<T> load(Callable<T> callable) {
try {
return new Result<>(callable.call());
} catch (Throwable e) {
return new Result<>(e);
}
}
private GitHub github;
private IOException connectionError;
private Collection<String> owners;
private Cache<String, Collection<String>> reposByOwner = CacheBuilder.newBuilder()
private Cache<String, Result<Collection<String>>> reposByOwner = CacheBuilder.newBuilder()
.expireAfterAccess(10, TimeUnit.MINUTES)
.build();
@@ -94,17 +125,23 @@ public class DefaultGithubInfoProvider implements GithubInfoProvider {
}
try {
if (github!=null) {
return reposByOwner.get(ownerName, () -> {
return reposByOwner.get(ownerName, loader(() -> {
GHPerson owner = getOwner(ownerName);
return Flux.fromIterable(owner.listRepositories())
.filter(repo -> repo.getOwnerName().equals(ownerName))
.map(GHRepository::getName)
.collect(CollectorUtil.toImmutableSet())
.block();
});
if (owner!=null) {
return Flux.fromIterable(owner.listRepositories())
.filter(repo -> repo.getOwnerName().equals(ownerName))
.map(GHRepository::getName)
.collect(CollectorUtil.toImmutableSet())
.block();
}
return null;
}))
.get();
}
} catch (Exception e) {
Log.log(e);
if (!isMissingOwnerException(e)) {
Log.log(e);
}
}
return ImmutableList.of();
}
@@ -112,9 +149,23 @@ public class DefaultGithubInfoProvider implements GithubInfoProvider {
private GHPerson getOwner(String ownerName) throws IOException {
try {
return github.getUser(ownerName);
} catch (IOException e) {
return github.getOrganization(ownerName);
} catch (IOException e1) {
if (isMissingOwnerException(e1)) {
try {
return github.getOrganization(ownerName);
} catch (IOException e2) {
if (isMissingOwnerException(e2)) {
return null;
}
throw e2;
}
}
throw e1;
}
}
protected boolean isMissingOwnerException(Throwable e) {
return e instanceof GHFileNotFoundException;
}
}

16
headless-services/concourse-language-server/src/main/java/org/springframework/ide/vscode/concourse/github/GithubInfoProvider.java
View File

@@ -17,12 +17,20 @@ public interface GithubInfoProvider {
/**
* Retrieves a list of owners that can be suggested as hints in completions of github repo urls.
* <p>
* Note: since github has millions of users and fetching all of them isn't really an option, the owners
* are expected to be somehow limited based on the user credentials (i.e. returning a list that is
* deemed relevant to the current logged in user, rather than an exhaustive list of every org and user name
* on github.
* Note: This list can not be used to determine whether a given owner exists. Github has millions of users
* and fetching all of them isn't really an option, therefore the owners are expected to be somehow limited
* based on the user credentials (i.e. returning a list that is deemed relevant to the current logged in user,
* rather than an exhaustive list of every org and user name on github.
*/
Collection<String> getOwners() throws Exception;
/**
* Fetch information about the repos owned by a given user or org.
* <p>
* Should return null rather than throw an exception for the case
* where the owner did not exist. The caller can make use of this
* to determine implicitly whether a given owner is valid.
*/
Collection<String> getReposForOwner(String owner) throws Exception;
}

16
headless-services/concourse-language-server/src/main/java/org/springframework/ide/vscode/concourse/github/GithubRepoContentAssistant.java
View File

@@ -13,25 +13,21 @@ package org.springframework.ide.vscode.concourse.github;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.eclipse.lsp4j.CompletionItemKind;
import org.springframework.ide.vscode.commons.languageserver.completion.ICompletionProposal;
import org.springframework.ide.vscode.commons.languageserver.completion.SimpleCompletionFactory;
import org.springframework.ide.vscode.commons.languageserver.util.DocumentRegion;
import org.springframework.ide.vscode.commons.util.Assert;
import org.springframework.ide.vscode.commons.util.ExceptionUtil;
import org.springframework.ide.vscode.commons.util.FuzzyMatcher;
import org.springframework.ide.vscode.commons.yaml.completion.CompletionFactory;
import org.springframework.ide.vscode.commons.yaml.schema.ISubCompletionEngine;
import org.eclipse.lsp4j.CompletionItemKind;
import com.google.common.collect.ImmutableList;
public class GithubRepoContentAssistant implements ISubCompletionEngine {
private String[] uriPrefixes = {
public static final String[] URI_PREFIXES = {
"git@github.com:",
"https://github.com/"
};
@@ -45,14 +41,14 @@ public class GithubRepoContentAssistant implements ISubCompletionEngine {
public List<ICompletionProposal> getCompletions(CompletionFactory f, DocumentRegion region, int offset) {
DocumentRegion query = region.subSequence(0, offset);
//If uri prefix is already there, we provide CA for owner / repo
for (String uriPrefix : uriPrefixes) {
for (String uriPrefix : URI_PREFIXES) {
if (query.startsWith(uriPrefix)) {
return getOwnerOrRepoCompletions(f, query.subSequence(uriPrefix.length()));
}
}
//If uri prefix is not yet there, maybe we can suggest it (if it matches the query)
List<ICompletionProposal> proposals = new ArrayList<>(uriPrefixes.length);
for (String uriPrefix : uriPrefixes) {
List<ICompletionProposal> proposals = new ArrayList<>(URI_PREFIXES.length);
for (String uriPrefix : URI_PREFIXES) {
if (FuzzyMatcher.matchScore(query, uriPrefix)!=0.0) {
proposals.add(SimpleCompletionFactory.simpleProposal(query, CompletionItemKind.Text, uriPrefix, null, null));
}
@@ -93,7 +89,7 @@ public class GithubRepoContentAssistant implements ISubCompletionEngine {
List<ICompletionProposal> proposals = new ArrayList<>(repos.size());
for (String repo : repos) {
if (FuzzyMatcher.matchScore(query, repo)!=0.0) {
proposals.add(SimpleCompletionFactory.simpleProposal(query, CompletionItemKind.Text, repo, null, null));
proposals.add(SimpleCompletionFactory.simpleProposal(query, CompletionItemKind.Text, repo+".git", null, null));
}
}
return proposals;

103
headless-services/concourse-language-server/src/main/java/org/springframework/ide/vscode/concourse/github/GithubValueParsers.java Normal file
View File

@@ -0,0 +1,103 @@
/*******************************************************************************
* Copyright (c) 2017 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Pivotal, Inc. - initial API and implementation
*******************************************************************************/
package org.springframework.ide.vscode.concourse.github;
import java.util.Collection;
import org.springframework.ide.vscode.commons.languageserver.reconcile.ReconcileException;
import org.springframework.ide.vscode.commons.languageserver.util.DocumentRegion;
import org.springframework.ide.vscode.commons.util.ValueParseException;
import org.springframework.ide.vscode.commons.util.ValueParser;
import org.springframework.ide.vscode.commons.util.text.LanguageId;
import org.springframework.ide.vscode.commons.util.text.TextDocument;
import org.springframework.ide.vscode.concourse.PipelineYmlSchemaProblems;
public class GithubValueParsers {
public static class GithubRepoReference {
public final DocumentRegion owner;
public final DocumentRegion name;
public GithubRepoReference(DocumentRegion owner, DocumentRegion name) {
super();
this.owner = owner;
this.name = name;
}
}
private static ReconcileException unknownEntity(String msg, DocumentRegion highlight) {
return new ReconcileException(msg, PipelineYmlSchemaProblems.UNKNOWN_GITHUB_ENTITY, highlight.getStart(), highlight.getEnd());
}
public static ValueParser uri(GithubInfoProvider github) {
return new ValueParser() {
@Override public GithubRepoReference parse(String _str) throws Exception {
TextDocument doc = new TextDocument(null, LanguageId.PLAINTEXT);
doc.setText(_str);
DocumentRegion str = new DocumentRegion(doc);
GithubRepoReference repo = parseFormat(str);
if (repo!=null) {
Collection<String> knownRepos;
try {
knownRepos = github.getReposForOwner(repo.owner.toString());
} catch (Exception e) {
//Couldn't read info from github. Ignore this in reconciler context.
return repo;
}
if (knownRepos==null) {
throw unknownEntity("User or Organization not found: '"+repo.owner+"'", repo.owner);
} else {
if (!knownRepos.contains(repo.name.toString())) {
throw unknownEntity("Repo not found: '"+repo.name+"'", repo.name);
}
}
}
return repo;
}
private GithubRepoReference parseFormat(DocumentRegion str) throws Exception {
String prefix = checkPrefix(str);
if (prefix!=null) {
DocumentRegion ownerAndName = str.subSequence(prefix.length());
//Should end with '.git'
if (ownerAndName.endsWith(".git")) {
ownerAndName = ownerAndName.subSequence(0, ownerAndName.length()-4);
} else {
DocumentRegion highlight = ownerAndName.textAtEnd(1);
throw new ValueParseException("GitHub repo uri should end with '.git'", highlight.getStart(), highlight.getEnd());
}
int slash = ownerAndName.indexOf('/');
if (slash>=0) {
return new GithubRepoReference(ownerAndName.subSequence(0, slash), ownerAndName.subSequence(slash+1));
} else {
throw new ValueParseException("Expecting something of the form '${owner}/${repo}'.", ownerAndName.getStart(), ownerAndName.getEnd());
}
}
return null;
}
private String checkPrefix(DocumentRegion str) throws ValueParseException {
for (String expectedPrefix : GithubRepoContentAssistant.URI_PREFIXES) {
int lastChar = expectedPrefix.length()-1;
if (str.startsWith(expectedPrefix.substring(0, lastChar))) {
if (str.charAt(lastChar)==expectedPrefix.charAt(lastChar)) {
return expectedPrefix;
}
throw new ValueParseException("Expecting a '"+expectedPrefix.charAt(lastChar)+"'", lastChar, lastChar+1);
}
}
return null;
}
};
}
}

158
headless-services/concourse-language-server/src/test/java/org/springframework/ide/vscode/concourse/ConcourseEditorTest.java
View File

@@ -12,6 +12,7 @@ package org.springframework.ide.vscode.concourse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.when;
import static org.springframework.ide.vscode.languageserver.testharness.Editor.INDENTED_COMPLETION;
import static org.springframework.ide.vscode.languageserver.testharness.Editor.PLAIN_COMPLETION;
@@ -30,18 +31,21 @@ import org.eclipse.lsp4j.DiagnosticSeverity;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.ide.vscode.commons.languageserver.reconcile.ProblemSeverity;
import org.springframework.ide.vscode.commons.util.IOUtil;
import org.springframework.ide.vscode.commons.util.Unicodes;
import org.springframework.ide.vscode.commons.util.text.LanguageId;
import org.springframework.ide.vscode.commons.yaml.completion.YamlCompletionEngineOptions;
import org.springframework.ide.vscode.commons.yaml.reconcile.YamlSchemaProblems;
import org.springframework.ide.vscode.concourse.github.GithubInfoProvider;
import org.springframework.ide.vscode.concourse.github.GithubRepoContentAssistant;
import org.springframework.ide.vscode.languageserver.testharness.CodeAction;
import org.springframework.ide.vscode.languageserver.testharness.Editor;
import org.springframework.ide.vscode.languageserver.testharness.LanguageServerHarness;
import org.springframework.ide.vscode.languageserver.testharness.SynchronizationPoint;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.mockito.Mockito;
@@ -220,6 +224,7 @@ public class ConcourseEditorTest {
public void reconcileAcceptsSensiblePipelineFile() throws Exception {
Editor editor;
when(github.getReposForOwner("spring-projects")).thenReturn(ImmutableSet.of("sts4"));
editor = harness.newEditor(
getClasspathResourceText("workspace/pipeline.yml")
);
@@ -643,7 +648,7 @@ public class ConcourseEditorTest {
"- name: sts4\n" +
" type: git\n" +
" source:\n" +
" uri: https://github.com/kdvolder/somestuff\n" +
" uri: https://someplace.com/kdvolder/somestuff\n" +
" branch: master\n" +
"jobs:\n" +
"- name: job1\n" +
@@ -848,14 +853,14 @@ public class ConcourseEditorTest {
" type: git\n" +
" source:\n" +
" branch: master\n" +
" uri: https://github.com/kdvolder/my-repo\n" +
" uri: https://someplace.com/kdvolder/my-repo\n" +
"resources:\n" +
"- name: your-repo\n" +
" type: git\n" +
" type: git\n" +
" source:\n" +
" branch: master\n" +
" uri: https://github.com/kdvolder/forked-repo\n"
" uri: https://someplace.com/kdvolder/forked-repo\n"
);
editor.assertProblems(
@@ -1049,7 +1054,7 @@ public class ConcourseEditorTest {
"- name: sts4-out\n" +
" type: git\n" +
" source:\n" +
" uri: git@github.com:spring-projects/sts4.git\n" +
" uri: git@someplace.com:spring-projects/sts4.git\n" +
" bogus: bad\n" +
" branch: {{branch}}\n" +
" private_key: {{rsa_id}}\n" +
@@ -1564,7 +1569,7 @@ public class ConcourseEditorTest {
"- name: repo\n" +
" type: git\n" +
" source:\n" +
" uri: git@github.com/johny-coder/test-repo\n" +
" uri: git@someplace.com:johny-coder/test-repo.git\n" +
"jobs:\n" +
"- name: do-stuff\n" +
" plan:\n" +
@@ -1580,7 +1585,7 @@ public class ConcourseEditorTest {
"- name: repo\n" +
" type: git\n" +
" source:\n" +
" uri: git@github.com/johny-coder/test-repo\n" +
" uri: git@github.com:johny-coder/test-repo\n" +
"jobs:\n" +
"- name: do-stuff\n" +
" plan:\n" +
@@ -3929,7 +3934,7 @@ public class ConcourseEditorTest {
" type: git\n" +
" source:\n" +
" branch: master\n" +
" uri: git@github.com/blah\n" +
" uri: git@someplace.com:blah/blah.git\n" +
"jobs:\n" +
"- name: build-it\n" +
" plan:\n" +
@@ -3949,7 +3954,7 @@ public class ConcourseEditorTest {
" - name: cf-networking-dev\n" +
" type: git\n" +
" source:\n" +
" uri: git@github.com:cloudfoundry-incubator/cf-networking-release.git\n" +
" uri: git@someplace.com:cloudfoundry-incubator/cf-networking-release.git\n" +
" branch: develop\n" +
" ignore_paths:\n" +
" - docs\n" +
@@ -4443,23 +4448,23 @@ public class ConcourseEditorTest {
" uri: <*>"
);
editor.assertContextualCompletions("https://github.com/the-owner/<*>",
"https://github.com/the-owner/cool-project<*>",
"https://github.com/the-owner/good-stuff<*>",
"https://github.com/the-owner/nice-repo<*>"
"https://github.com/the-owner/cool-project.git<*>",
"https://github.com/the-owner/good-stuff.git<*>",
"https://github.com/the-owner/nice-repo.git<*>"
);
editor.assertContextualCompletions("https://github.com/the-owner/proj<*>",
"https://github.com/the-owner/cool-project<*>"
"https://github.com/the-owner/cool-project.git<*>"
);
editor.assertContextualCompletions("git@github.com:the-owner/<*>",
"git@github.com:the-owner/cool-project<*>",
"git@github.com:the-owner/good-stuff<*>",
"git@github.com:the-owner/nice-repo<*>"
"git@github.com:the-owner/cool-project.git<*>",
"git@github.com:the-owner/good-stuff.git<*>",
"git@github.com:the-owner/nice-repo.git<*>"
);
editor.assertContextualCompletions("git@github.com:the-owner/proj<*>",
"git@github.com:the-owner/cool-project<*>"
"git@github.com:the-owner/cool-project.git<*>"
);
}
@@ -4485,6 +4490,127 @@ public class ConcourseEditorTest {
editor.assertCompletionLabels("Explain some stuff");
}
@Test public void noNetworkGithubUriReconciling() throws Exception {
//When not plugged into the network (i.e github api returns errors)
// we consider the repos unknowable and will not warn about anything.
when(github.getReposForOwner("the-owner")).thenThrow(new Exception("Some problem talking to github"));
Editor editor = harness.newEditor(
"resources:\n" +
"- name: my-repo\n" +
" type: git\n" +
" source:\n" +
" uri: git@github.com:the-owner/bad-project.git\n" +
"- name: other-repo\n" +
" type: git\n" +
" source:\n" +
" uri: https://github.com/the-owner/wrong-project.git\n"
);
editor.assertProblems(
"my-repo|Unused",
"other-repo|Unused"
);
}
@Test public void nonGithubUriReconciling() throws Exception {
//We only care about gihub uris. So, ignore anything else for reconciler.
when(github.getReposForOwner("the-owner")).thenReturn(ImmutableList.of(
"nice-repo", "cool-project", "good-stuff"
));
Editor editor = harness.newEditor(
"resources:\n" +
"- name: my-repo\n" +
" type: git\n" +
" source:\n" +
" uri: git@somehost.somewhere:the-owner/bad-project\n" +
"- name: other-repo\n" +
" type: git\n" +
" source:\n" +
" uri: https://somehost.somewhere/the-owner/wrong-project\n"
);
editor.assertProblems(
"my-repo|Unused",
"other-repo|Unused"
);
}
@Test public void githubWrongFormatReconciling() throws Exception {
when(github.getReposForOwner("the-owner")).thenReturn(ImmutableList.of(
"nice-repo", "cool-project", "good-stuff"
));
Editor editor = harness.newEditor(
"resources:\n" +
"- name: my-repo\n" +
" type: git\n" +
" source:\n" +
" uri: git@github.com:ssh-just-owner\n" +
"- name: my-repo2\n" +
" type: git\n" +
" source:\n" +
" uri: git@github.com:ssh-just-owner.git\n" +
"- name: other-repo\n" +
" type: git\n" +
" source:\n" +
" uri: https://github.com/https-just-owner\n" +
"- name: other-repo2\n" +
" type: git\n" +
" source:\n" +
" uri: https://github.com/https-just-owner.git\n" +
"- name: different-repo\n" +
" type: git\n" +
" source:\n" +
" uri: git@github.com/ssh/wrong-separator.git\n" +
"- name: one-more-repo\n" +
" type: git\n" +
" source:\n" +
" uri: https://github.com:https/wrong-separator.git\n"
);
editor.ignoreProblem(PipelineYmlSchemaProblems.UNUSED_RESOURCE);
editor.assertProblems(
"uri: git@github.com:ssh-just-owne^r^\n|should end with '.git'",
"ssh-just-owner|Expecting something of the form '${owner}/${repo}'",
"uri: https://github.com/https-just-owne^r^\n|should end with '.git'",
"https-just-owner|Expecting something of the form '${owner}/${repo}'",
"git@github.com^/^ssh/wrong-separator|Expecting a ':'",
"https://github.com^:^https/wrong-separator|Expecting a '/'"
);
}
@Test public void githubUriReconciling() throws Exception {
when(github.getReposForOwner("the-owner")).thenReturn(ImmutableList.of(
"nice-repo", "cool-project", "good-stuff"
));
when(github.getReposForOwner("owner-no-exist")).thenReturn(null);
String editorText =
"resources:\n" +
"- name: my-repo\n" +
" type: git\n" +
" source:\n" +
" uri: $$github$$the-owner/cool-project.git\n" +
"- name: other-repo\n" +
" type: git\n" +
" source:\n" +
" uri: $$github$$the-owner/repo-no-exist.git\n" +
"- name: different-repo\n" +
" type: git\n" +
" source:\n" +
" uri: $$github$$owner-no-exist/who-cares.git\n"
;
for (String githubUriPrefix : GithubRepoContentAssistant.URI_PREFIXES) {
Editor editor = harness.newEditor(editorText.replace("$$github$$", githubUriPrefix));
editor.ignoreProblem(PipelineYmlSchemaProblems.UNUSED_RESOURCE);
List<Diagnostic> problems = editor.assertProblems(
"repo-no-exist|Repo not found: 'repo-no-exist'",
"owner-no-exist|User or Organization not found: 'owner-no-exist'"
);
for (Diagnostic d : problems) {
assertEquals(DiagnosticSeverity.Warning, d.getSeverity());
}
}
}
//////////////////////////////////////////////////////////////////////////////
private void assertContextualCompletions(String conText, String textBefore, String... textAfter) throws Exception {