Concourse: reconcile checking for non-existent resources

vscode-extensions/vscode-concourse/src/main/java/org/springframework/ide/vscode/concourse/ConcourseLanguageServer.java

@@ -1,3 +1,13 @@
+/*******************************************************************************
+ * Copyright (c) 2016 Pivotal, Inc.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Pivotal, Inc. - initial API and implementation
+ *******************************************************************************/
 package org.springframework.ide.vscode.concourse;
 
 import org.eclipse.lsp4j.CompletionOptions;
@@ -25,22 +35,20 @@ import org.yaml.snakeyaml.Yaml;
 
 public class ConcourseLanguageServer extends SimpleLanguageServer {
 
-	private Yaml yaml = new Yaml();
-	private YamlSchema schema = new PipelineYmlSchema();
-
-	
 	public ConcourseLanguageServer() {
 		SimpleTextDocumentService documents = getTextDocumentService();
-		
-		YamlASTProvider parser = new YamlParser(yaml);
+
+		ConcourseModel models = new ConcourseModel(documents);
+		YamlASTProvider currentAsts = models.getAstProvider(false);
 
 		YamlStructureProvider structureProvider = YamlStructureProvider.DEFAULT;
+		YamlSchema schema = new PipelineYmlSchema(models);
 		YamlAssistContextProvider contextProvider = new SchemaBasedYamlAssistContextProvider(schema);
 		YamlCompletionEngine yamlCompletionEngine = new YamlCompletionEngine(structureProvider, contextProvider);
 		VscodeCompletionEngine completionEngine = new VscodeCompletionEngineAdapter(this, yamlCompletionEngine);
-		HoverInfoProvider infoProvider = new YamlHoverInfoProvider(parser, structureProvider, contextProvider);
+		HoverInfoProvider infoProvider = new YamlHoverInfoProvider(currentAsts, structureProvider, contextProvider);
 		VscodeHoverEngine hoverEngine = new VscodeHoverEngineAdapter(this, infoProvider);
-		IReconcileEngine engine = new YamlSchemaBasedReconcileEngine(parser, schema);
+		IReconcileEngine engine = new YamlSchemaBasedReconcileEngine(currentAsts, schema);
 
 //		SimpleWorkspaceService workspace = getWorkspaceService();
 		documents.onDidChangeContent(params -> {

vscode-extensions/vscode-concourse/src/main/java/org/springframework/ide/vscode/concourse/ConcourseModel.java

@@ -0,0 +1,109 @@
+/*******************************************************************************
+ * Copyright (c) 2016 Pivotal, Inc.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Pivotal, Inc. - initial API and implementation
+ *******************************************************************************/
+package org.springframework.ide.vscode.concourse;
+
+import static org.springframework.ide.vscode.commons.yaml.path.YamlPathSegment.anyChild;
+import static org.springframework.ide.vscode.commons.yaml.path.YamlPathSegment.valueAt;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.ide.vscode.commons.languageserver.util.SimpleTextDocumentService;
+import org.springframework.ide.vscode.commons.languageserver.util.TextDocumentContentChange;
+import org.springframework.ide.vscode.commons.util.Log;
+import org.springframework.ide.vscode.commons.util.text.IDocument;
+import org.springframework.ide.vscode.commons.yaml.ast.NodeUtil;
+import org.springframework.ide.vscode.commons.yaml.ast.YamlASTProvider;
+import org.springframework.ide.vscode.commons.yaml.ast.YamlFileAST;
+import org.springframework.ide.vscode.commons.yaml.ast.YamlParser;
+import org.springframework.ide.vscode.commons.yaml.path.YamlPath;
+import org.springframework.ide.vscode.concourse.util.StaleFallbackCache;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.error.YAMLException;
+import org.yaml.snakeyaml.nodes.Node;
+
+/**
+ * ConcourseModels is responsible for extracting various bits of information 
+ * out of .yml documents and caching them for use by various tools (reconcile engine
+ * and completion engine).
+ */
+public class ConcourseModel {
+	
+	private static final YamlPath RESOURCE_NAMES_PATH = new YamlPath(
+		valueAt("resources"),
+		anyChild(),
+		valueAt("name")
+	);
+	
+	private final YamlParser parser;
+	private StaleFallbackCache<String, YamlFileAST> asts = new StaleFallbackCache<>();
+
+	public ConcourseModel(SimpleTextDocumentService documents) {
+		Yaml yaml = new Yaml();
+		this.parser = new YamlParser(yaml);
+		documents.onDidChangeContent(this::documentChanged);
+	}
+
+	private void documentChanged(TextDocumentContentChange changeEvent) {
+		String uri = changeEvent.getDocument().getUri();
+		if (uri!=null) {
+			asts.invalidate(uri);
+		}
+	}
+	
+	/**
+	 * Returns the resource names that are defined by given IDocument. If the contents
+	 * of IDocument is not currently parseable then this may return stale information
+	 * retained from a previous successful parse.
+	 * <p>
+	 * It may also return null if its not currently possible to obtain the list of resource
+	 * names (e.g. because there hasn't been a successful parse yet and current document contents
+	 * can not be parsed).
+	 */
+	public Set<String> getResourceNames(IDocument doc) {
+		try {
+			if (doc!=null) {
+				String uri = doc.getUri();
+				if (uri!=null) {
+					YamlFileAST ast = getAst(doc);
+					Node root = ast.get(0);
+					return RESOURCE_NAMES_PATH
+						.traverseAmbiguously(root)
+						.map(NodeUtil::asScalar)
+						.filter((string) -> string!=null)
+						.collect(Collectors.toSet());
+				}
+			}
+		} catch (YAMLException e) {
+			// ignore: garbage in the doc. Can't compute stuff and that's to be expected.
+		} catch (Exception e) {
+			Log.log(e);
+		}
+		return null;
+	}
+
+	private YamlFileAST getAst(IDocument doc) throws Exception {
+		return getAstProvider(true).getAST(doc);
+	}
+
+	public YamlASTProvider getAstProvider(boolean allowStaleAsts) {
+		return (IDocument doc) -> {
+			String uri = doc.getUri();
+			if (uri!=null) {
+				return asts.get(uri, allowStaleAsts, () -> {
+					return parser.getAST(doc);
+				});
+			}
+			return null;
+		};
+	}
+
+}

vscode-extensions/vscode-concourse/src/main/java/org/springframework/ide/vscode/concourse/PipelineYmlSchema.java

@@ -12,6 +12,7 @@ package org.springframework.ide.vscode.concourse;
 
 import org.springframework.ide.vscode.commons.util.Renderable;
 import org.springframework.ide.vscode.commons.util.Renderables;
+import org.springframework.ide.vscode.commons.yaml.schema.DynamicSchemaContext;
 import org.springframework.ide.vscode.commons.yaml.schema.YType;
 import org.springframework.ide.vscode.commons.yaml.schema.YTypeFactory;
 import org.springframework.ide.vscode.commons.yaml.schema.YTypeFactory.AbstractType;
@@ -31,7 +32,7 @@ public class PipelineYmlSchema implements YamlSchema {
 
 	private final YTypeFactory f = new YTypeFactory();
 
-	public PipelineYmlSchema() {
+	public PipelineYmlSchema(ConcourseModel models) {
 		TYPE_UTIL = f.TYPE_UTIL;
 
 		// define schema types
@@ -81,9 +82,18 @@ public class PipelineYmlSchema implements YamlSchema {
 //
 //				 The vagrant-cloud r
 		);
+		
+		YType resourceName = f.yenum("ResourceName", 
+				(parseString, validValues) ->  {
+					return "The '"+parseString+"' resource does not exist. Existing resources: "+validValues;
+				},
+				(DynamicSchemaContext dc) -> {
+					return models.getResourceNames(dc.getDocument());
+				}
+		);
 
 		YBeanType getStep = f.ybean("GetStep");
-		prop(getStep, "get", t_ne_string);
+		prop(getStep, "get", resourceName);
 		prop(getStep, "resource", t_string);
 		prop(getStep, "version", t_version);
 		prop(getStep, "passed", t_strings);
@@ -91,7 +101,7 @@ public class PipelineYmlSchema implements YamlSchema {
 		prop(getStep, "trigger", t_boolean);
 
 		YBeanType putStep = f.ybean("PutStep");
-		prop(putStep, "put", t_ne_string);
+		prop(putStep, "put", resourceName);
 		prop(putStep, "resource", t_string);
 		prop(putStep, "params", t_params);
 		prop(putStep, "get_params", t_params);

vscode-extensions/vscode-concourse/src/main/java/org/springframework/ide/vscode/concourse/util/StaleFallbackCache.java

@@ -0,0 +1,91 @@
+/*******************************************************************************
+ * Copyright (c) 2016 Pivotal, Inc.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Pivotal, Inc. - initial API and implementation
+ *******************************************************************************/
+package org.springframework.ide.vscode.concourse.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import org.springframework.ide.vscode.commons.util.Assert;
+import org.springframework.ide.vscode.commons.util.ExceptionUtil;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+/**
+ * Simple cache implementation that falls back on 'stale' cache entry if
+ * a new entry can not be computed. The api is loosely modeled after 
+ * guava's Cache interface (but only the subset we use is implemented to reduce the
+ * complexity of its implementation).
+ */
+public class StaleFallbackCache<K, V>{
+
+	Map<K, V> staleEntries = new HashMap<>();
+	Cache<K, CompletableFuture<V>> validEntries = CacheBuilder.newBuilder().build();
+
+
+	public synchronized V get(K key, boolean allowStaleEntries, Callable<? extends V> valueLoader) throws Exception {
+		CompletableFuture<V> valid = validEntries.get(key, () -> load(valueLoader));
+		if (!allowStaleEntries) {
+			return future_get(valid);
+		} else {
+			if (valid.isCompletedExceptionally()) {
+				V staleValue = staleEntries.get(key);
+				if (staleValue!=null) {
+					return staleValue;
+				}
+			}
+			return future_get(valid);
+		}
+	}
+	
+	public synchronized void invalidate(K key) {
+		CompletableFuture<V> staleEntry = validEntries.getIfPresent(key);
+		if (staleEntry!=null) {
+			validEntries.invalidate(key);
+			try {
+				staleEntries.put(key, future_get(staleEntry));
+			} catch (Exception e) {
+				//ignore. Don't overwrite stale entry if current entry represents an error.
+				// We only keep 'good quality' stale entries not failed attempts to compute a value.
+				// as it is kind of the point to fall back on a 'good' old entry when the current
+				// entry is unavailable because of a problem (e.g. problems parsing the AST).
+			}
+		}
+	}
+
+
+
+	private V future_get(CompletableFuture<V> f) throws Exception {
+		try {
+			return f.get();
+		} catch (InterruptedException e) {
+			throw e;
+		} catch (ExecutionException e) {
+			throw ExceptionUtil.exception(e.getCause());
+		}
+	}
+
+	private CompletableFuture<V> load(Callable<? extends V> valueLoader) {
+		CompletableFuture<V> future = new CompletableFuture<V>();
+		try {
+			V value = valueLoader.call();
+			Assert.isNotNull(value);
+			future.complete(value);
+		} catch (Throwable e) {
+			future.completeExceptionally(e);
+		}
+		return future;
+	}
+
+}

vscode-extensions/vscode-concourse/src/test/java/org/springframework/ide/vscode/concourse/ConcourseLanguageServerTest.java

@@ -1,4 +1,4 @@
-package org.springframework.ide.vscode.manifest.yaml;
+package org.springframework.ide.vscode.concourse;
 
 import static org.assertj.core.api.Assertions.assertThat;
 

vscode-extensions/vscode-concourse/src/test/java/org/springframework/ide/vscode/concourse/PipelineYamlEditorTest.java

@@ -8,7 +8,7 @@
  * Contributors:
  *     Pivotal, Inc. - initial API and implementation
  *******************************************************************************/
-package org.springframework.ide.vscode.manifest.yaml;
+package org.springframework.ide.vscode.concourse;
 
 import static org.springframework.ide.vscode.languageserver.testharness.TestAsserts.assertContains;
 
@@ -259,12 +259,15 @@ public class PipelineYamlEditorTest {
 		);
 		editor.assertProblems(
 				"boohoo|boolean",
-				"-1|Positive Integer",
+				"-1|must be positive",
+				"git|resource does not exist",
 				"yohoho|boolean"
 		);
 
 		//check that correct values are indeed accepted
 		editor = harness.newEditor(
+				"resources:\n" +
+				"- name: git\n" +
 				"jobs:\n" +
 				"- name: foo\n" +
 				"  serial: true\n" +
@@ -379,6 +382,33 @@ public class PipelineYamlEditorTest {
 		editor.assertHoverContains("jobs", "At a high level, a job describes some actions to perform");
 	}
 	
+	@Test
+	public void reconcileResourceReferences() throws Exception {
+		Editor editor = harness.newEditor(
+				"resources:\n" + 
+				"- name: sts4\n" + 
+				"  type: git\n" + 
+				"  source:\n" + 
+				"    repository: https://github.com/kdvolder/somestuff\n" + 
+				"jobs:\n" + 
+				"- name: job1\n" + 
+				"  plan:\n" + 
+				"  - get: sts4\n" + 
+				"  - get: bogus-get\n" + 
+				"  - put: bogus-put\n"
+		);
+		editor.assertProblems(
+				"bogus-get|resource does not exist",
+				"bogus-put|resource does not exist"
+		);
+		
+		editor.assertProblems(
+				"bogus-get|[sts4]",
+				"bogus-put|[sts4]"
+		);
+
+	}
+
 	//////////////////////////////////////////////////////////////////////////////
 
 	private void assertContextualCompletions(String conText, String textBefore, String... textAfter) throws Exception {

vscode-extensions/vscode-concourse/src/test/java/org/springframework/ide/vscode/manifest/yaml/PipelineYmlSchemaTest.java

@@ -1,129 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2015, 2016 Pivotal, Inc.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- *     Pivotal, Inc. - initial API and implementation
- *******************************************************************************/
-package org.springframework.ide.vscode.manifest.yaml;
-
-import static org.junit.Assert.fail;
-
-import org.junit.Test;
-import org.springframework.ide.vscode.concourse.PipelineYmlSchema;
-
-/**
- * @author Kris De Volder
- */
-public class PipelineYmlSchemaTest {
-
-//	@Test
-//	public void shouldMakeSomeTests() {
-//		fail("We should make some tests for this");
-//	}
-//	
-//	private static final String[] NESTED_PROP_NAMES = {
-////			"applications",
-//			"buildpack",
-//			"command",
-//			"disk_quota",
-//			"domain",
-//			"domains",
-//			"env",
-//			"health-check-type",
-//			"host",
-//			"hosts",
-////			"inherit",
-//			"instances",
-//			"memory",
-//			"name",
-//			"no-hostname",
-//			"no-route",
-//			"path",
-//			"random-route",
-//			"services",
-//			"stack",
-//			"timeout"
-//	};
-//
-	private static final String[] TOPLEVEL_PROP_NAMES = {
-			"resources",
-			"jobs",
-			"resource-types"
-			//groups
-	};
-
-	PipelineYmlSchema schema = new PipelineYmlSchema();
-//
-//	@Test
-//	public void toplevelProperties() throws Exception {
-//		assertPropNames(schema.getTopLevelType().getProperties(DynamicSchemaContext.NULL), TOPLEVEL_PROP_NAMES);
-//		assertPropNames(schema.getTopLevelType().getPropertiesMap(DynamicSchemaContext.NULL), TOPLEVEL_PROP_NAMES);
-//	}
-//
-//	@Test
-//	public void nestedProperties() throws Exception {
-//		assertPropNames(getNestedProps(), NESTED_PROP_NAMES);
-//	}
-//
-//	@Test
-//	public void toplevelPropertiesHaveDescriptions() {
-//		for (YTypedProperty p : schema.getTopLevelType().getProperties(DynamicSchemaContext.NULL)) {
-//			if (!p.getName().equals("applications")) {
-//				assertHasRealDescription(p);
-//			}
-//		}
-//	}
-//
-//	@Test
-//	public void nestedPropertiesHaveDescriptions() {
-//		for (YTypedProperty p : getNestedProps()) {
-//			assertHasRealDescription(p);
-//		}
-//	}
-//
-//	//////////////////////////////////////////////////////////////////////////////
-//
-//	private void assertHasRealDescription(YTypedProperty p) {
-//		{
-//			String noDescriptionText = Renderables.NO_DESCRIPTION.toHtml();
-//			String actual = p.getDescription().toHtml();
-//			String msg = "Description missing for '"+p.getName()+"'";
-//			assertTrue(msg, StringUtil.hasText(actual));
-//			assertFalse(msg, noDescriptionText.equals(actual));
-//		}
-//		{
-//			String noDescriptionText = Renderables.NO_DESCRIPTION.toMarkdown();
-//			String actual = p.getDescription().toMarkdown();
-//			String msg = "Description missing for '"+p.getName()+"'";
-//			assertTrue(msg, StringUtil.hasText(actual));
-//			assertFalse(msg, noDescriptionText.equals(actual));
-//		}
-//	}
-//
-//	private List<YTypedProperty> getNestedProps() {
-//		YSeqType applications = (YSeqType) schema.getTopLevelType().getPropertiesMap().get("applications").getType();
-//		YBeanType application = (YBeanType) applications.getDomainType();
-//		return application.getProperties();
-//	}
-//
-//	private void assertPropNames(List<YTypedProperty> properties, String... expectedNames) {
-//		assertEquals(ImmutableSet.copyOf(expectedNames), getNames(properties));
-//	}
-//
-//	private void assertPropNames(Map<String, YTypedProperty> propertiesMap, String[] toplevelPropNames) {
-//		assertEquals(ImmutableSet.copyOf(toplevelPropNames), ImmutableSet.copyOf(propertiesMap.keySet()));
-//	}
-//
-//	private ImmutableSet<String> getNames(Iterable<YTypedProperty> properties) {
-//		Builder<String> builder = ImmutableSet.builder();
-//		for (YTypedProperty p : properties) {
-//			builder.add(p.getName());
-//		}
-//		return builder.build();
-//	}
-
-}

vscode-extensions/vscode-concourse/src/test/resources/workspace/pipeline-with-bad-resources.yml

@@ -0,0 +1,11 @@
+resources:
+- name: sts4
+  type: git
+  source:
+    repository: https://github.com/kdvolder/somestuff
+jobs:
+- name: job1
+  plan:
+  - get: sts4
+  - get: bogus-get
+  - put: bogus-put