From 7d4766209472c0ef03b89d8e2b1cd43f31ccdcfe Mon Sep 17 00:00:00 2001 From: Kris De Volder Date: Tue, 5 Feb 2019 14:22:46 -0800 Subject: [PATCH] Reconciler support for yaml anchors, references and merges --- .../commons/commons-yaml/pom.xml | 1 - .../vscode/commons/yaml/ast/AstDumper.java | 50 ++++ .../commons/yaml/ast/NodeMergeSupport.java | 130 +++++++++++ .../SchemaBasedYamlASTReconciler.java | 31 ++- .../yaml/reconcile/YamlSchemaProblems.java | 2 - headless-services/commons/pom.xml | 1 - .../vscode/concourse/ConcourseEditorTest.java | 218 ++++++++++++++++++ 7 files changed, 427 insertions(+), 6 deletions(-) create mode 100644 headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/AstDumper.java create mode 100644 headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/NodeMergeSupport.java diff --git a/headless-services/commons/commons-yaml/pom.xml b/headless-services/commons/commons-yaml/pom.xml index 6a0549c41..7a1058143 100644 --- a/headless-services/commons/commons-yaml/pom.xml +++ b/headless-services/commons/commons-yaml/pom.xml @@ -26,7 +26,6 @@ org.yaml snakeyaml - ${yaml-version} javax.inject diff --git a/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/AstDumper.java b/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/AstDumper.java new file mode 100644 index 000000000..a16a2c4e6 --- /dev/null +++ b/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/AstDumper.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright (c) 2019 Pivotal, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Pivotal, Inc. - initial API and implementation + *******************************************************************************/ +package org.springframework.ide.vscode.commons.yaml.ast; + +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; + +public class AstDumper { + + public static void dump(Node node, int indent) { + if (node instanceof MappingNode) { + for (NodeTuple entry : ((MappingNode)node).getValue()) { + println(indent, NodeUtil.asScalar(entry.getKeyNode())+":"); + dump(entry.getValueNode(), indent+1); + } + } else if (node instanceof SequenceNode) { + for (Node el : ((SequenceNode)node).getValue()) { + println(indent, "["); + dump(el, indent+1); + println(indent, "]"); + } + } else if (node instanceof ScalarNode) { + println(indent, NodeUtil.asScalar(node)); + } else { + println(indent, "???"+node.getClass().getSimpleName()); + } + } + + private static void println(int indent, String string) { + indent(indent); + System.out.println(string); + } + + private static void indent(int indent) { + for (int i = 0; i < indent; i++) { + System.out.print(" "); + } + } +} diff --git a/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/NodeMergeSupport.java b/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/NodeMergeSupport.java new file mode 100644 index 000000000..59320b1c3 --- /dev/null +++ b/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/NodeMergeSupport.java @@ -0,0 +1,130 @@ +/** + * Copyright (c) 2008, 2019 http://www.snakeyaml.org and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Contributors: + * + * 2008 - http://www.snakeyaml.org original api and implementation. + * 2019 - Copied and modified by Pivotal. + */ +package org.springframework.ide.vscode.commons.yaml.ast; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.springframework.ide.vscode.commons.languageserver.reconcile.IProblemCollector; +import org.springframework.ide.vscode.commons.yaml.reconcile.YamlSchemaProblems; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.SequenceNode; +import org.yaml.snakeyaml.nodes.Tag; + +/** + * Bits and pieces copied from snakeyaml library to support dealing with '<<' merge + * nodes. + *

+ * Some modifications made to support our own use-cases. + */ +public class NodeMergeSupport { + + private IProblemCollector problems; + + public NodeMergeSupport(IProblemCollector problems) { + this.problems = problems; + } + + public void flattenMapping(MappingNode node) { + // perform merging only on nodes containing merge node(s) + //processDuplicateKeys(node); + if (node.isMerged()) { + node.setValue(mergeNode(node, true, new HashMap(), + new ArrayList())); + node.setMerged(false); + } + } + + /** + * Does merge for supplied mapping node. + * + * @param node + * where to merge + * @param isPreffered + * true if keys of node should take precedence over others... + * @param key2index + * maps already merged keys to index from values + * @param values + * collects merged NodeTuple + * @return list of the merged NodeTuple (to be set as value for the + * MappingNode) + */ + private List mergeNode(MappingNode node, boolean isPreffered, + Map key2index, List values) { + Iterator iter = node.getValue().iterator(); + while (iter.hasNext()) { + final NodeTuple nodeTuple = iter.next(); + final Node keyNode = nodeTuple.getKeyNode(); + final Node valueNode = nodeTuple.getValueNode(); + if (keyNode.getTag().equals(Tag.MERGE)) { + iter.remove(); + switch (valueNode.getNodeId()) { + case mapping: + MappingNode mn = (MappingNode) valueNode; + mergeNode(mn, false, key2index, values); + break; + case sequence: + SequenceNode sn = (SequenceNode) valueNode; + List vals = sn.getValue(); + for (Node subnode : vals) { + if (!(subnode instanceof MappingNode)) { + problems.accept(YamlSchemaProblems.schemaProblem( + "Expected a mapping for merging, but found "+subnode.getNodeId(), subnode + )); + } else { + MappingNode mnode = (MappingNode) subnode; + mergeNode(mnode, false, key2index, values); + } + } + break; + default: + problems.accept(YamlSchemaProblems.schemaProblem( + "Expected a mapping or list of mappings for merging, but found " + + valueNode.getNodeId(), + valueNode + )); + } + } else { + // we need to construct keys to avoid duplications + String key = NodeUtil.asScalar(keyNode); + if (key!=null) { + if (!key2index.containsKey(key)) { // 1st time merging key + values.add(nodeTuple); + // keep track where tuple for the key is + key2index.put(key, values.size() - 1); + } else if (isPreffered) { // there is value for the key, but we + // need to override it + // change value for the key using saved position + values.set(key2index.get(key), nodeTuple); + } + } + } + } + return values; + } + + +} diff --git a/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/SchemaBasedYamlASTReconciler.java b/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/SchemaBasedYamlASTReconciler.java index 9e798d42d..bd95a4ccb 100644 --- a/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/SchemaBasedYamlASTReconciler.java +++ b/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/SchemaBasedYamlASTReconciler.java @@ -40,6 +40,7 @@ import org.springframework.ide.vscode.commons.util.ValueParseException; import org.springframework.ide.vscode.commons.util.ValueParser; import org.springframework.ide.vscode.commons.util.text.DocumentRegion; import org.springframework.ide.vscode.commons.util.text.IDocument; +import org.springframework.ide.vscode.commons.yaml.ast.NodeMergeSupport; import org.springframework.ide.vscode.commons.yaml.ast.NodeUtil; import org.springframework.ide.vscode.commons.yaml.ast.YamlFileAST; import org.springframework.ide.vscode.commons.yaml.path.YamlPath; @@ -69,6 +70,7 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler { private final YTypeUtil typeUtil; private final ITypeCollector typeCollector; private final YamlQuickfixes quickfixes; + private final NodeMergeSupport nodeMerger; private List delayedConstraints = new ArrayList<>(); // keeps track of dynamic constraints discovered during reconciler walk @@ -84,6 +86,7 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler { this.typeCollector = typeCollector; this.typeUtil = schema.getTypeUtil(); this.quickfixes = quickfixes; + this.nodeMerger = new NodeMergeSupport(problems); } @Override @@ -138,7 +141,6 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler { } private void reconcile(YamlFileAST ast, YamlPath path, Node parent, Node node, YType _type) { -// IDocument doc = ast.getDocument(); if (_type!=null && !skipReconciling(node)) { DynamicSchemaContext schemaContext = new ASTDynamicSchemaContext(ast, path, node); YType type = typeUtil.inferMoreSpecificType(_type, schemaContext); @@ -149,6 +151,7 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler { switch (getNodeId(node)) { case mapping: MappingNode map = (MappingNode) node; + nodeMerger.flattenMapping(map); checkForDuplicateKeys(map); if (typeUtil.isMap(type)) { for (NodeTuple entry : map.getValue()) { @@ -167,7 +170,9 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler { } else { YTypedProperty prop = beanProperties.get(key); if (prop==null) { - unknownBeanProperty(keyNode, type, key); + if (!isAnchored(entry)) { + unknownBeanProperty(keyNode, type, key); + } } else { if (prop.isDeprecated()) { String msg = prop.getDeprecationMessage(); @@ -223,6 +228,28 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler { } } + /** + * Detects whether a given map key-value pair is anchored. I.e. corresponds to + * a bit of yaml like this example: + * + * 

+	 * some-key: &some-anchor
+	 *   blah: blah
+	 *   more: blah
+	 *
+ * + * @param entry + * @return + */ + private boolean isAnchored(NodeTuple entry) { + if (entry!=null) { + Node v = entry.getValueNode(); + String a = v.getAnchor(); + return a!=null; + } + return false; + } + private void parse(YamlFileAST ast, Node node, YType type, ValueParser parser) { try { String value = NodeUtil.asScalar(node); diff --git a/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/YamlSchemaProblems.java b/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/YamlSchemaProblems.java index 9bf854042..63d2cbfad 100644 --- a/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/YamlSchemaProblems.java +++ b/headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/YamlSchemaProblems.java @@ -23,7 +23,6 @@ import org.springframework.ide.vscode.commons.languageserver.reconcile.ProblemTy import org.springframework.ide.vscode.commons.languageserver.reconcile.ReconcileProblem; import org.springframework.ide.vscode.commons.languageserver.reconcile.ReconcileProblemImpl; import org.springframework.ide.vscode.commons.util.CollectionUtil; -import org.springframework.ide.vscode.commons.util.CollectorUtil; import org.springframework.ide.vscode.commons.util.text.DocumentRegion; import org.springframework.ide.vscode.commons.util.text.IDocument; import org.springframework.ide.vscode.commons.yaml.ast.NodeUtil; @@ -32,7 +31,6 @@ import org.springframework.ide.vscode.commons.yaml.path.YamlPathSegment; import org.springframework.ide.vscode.commons.yaml.schema.DynamicSchemaContext; import org.springframework.ide.vscode.commons.yaml.schema.YType; import org.springframework.ide.vscode.commons.yaml.schema.YTypedProperty; -import org.springframework.ide.vscode.commons.yaml.snippet.Snippet; import org.yaml.snakeyaml.error.Mark; import org.yaml.snakeyaml.nodes.MappingNode; import org.yaml.snakeyaml.nodes.Node; diff --git a/headless-services/commons/pom.xml b/headless-services/commons/pom.xml index 5f3882915..900a738b7 100644 --- a/headless-services/commons/pom.xml +++ b/headless-services/commons/pom.xml @@ -79,7 +79,6 @@ UTF-8 - 1.17 4.12 3.5.2 1.7.25 diff --git a/headless-services/concourse-language-server/src/test/java/org/springframework/ide/vscode/concourse/ConcourseEditorTest.java b/headless-services/concourse-language-server/src/test/java/org/springframework/ide/vscode/concourse/ConcourseEditorTest.java index 0108bff9e..04ddb3749 100644 --- a/headless-services/concourse-language-server/src/test/java/org/springframework/ide/vscode/concourse/ConcourseEditorTest.java +++ b/headless-services/concourse-language-server/src/test/java/org/springframework/ide/vscode/concourse/ConcourseEditorTest.java @@ -732,6 +732,7 @@ public class ConcourseEditorTest { ); } + @Test public void reconcileDuplicateResourceTypeNames_exempt_Builtins() throws Exception { //See https://github.com/spring-projects/sts4/issues/196 @@ -4882,6 +4883,223 @@ public class ConcourseEditorTest { editor.assertHoverContains("optional", "If `true`, then the input is not required by the task"); } + @Test + public void anchorNodeSuppressesUnknownPropertyError() throws Exception { + Editor editor = harness.newEditor( + "pool-template: &pool-template\n" + + " uri: ((pool-git-backing-store-uri))\n" + + " branch: master\n" + + " pool: OVERRIDEME\n" + + " private_key: ((pool-git-backing-store-private-key))\n" + ); + editor.assertProblems(/*none*/); + } + + @Test + public void referencedAnchorNodesReconciled() throws Exception { + Editor editor = harness.newEditor( + "repo-dflts: &repo-dflts\n" + + " bogus: bar\n" + + "resources:\n" + + "- name: foo\n" + + " type: git\n" + + " source: *repo-dflts\n" + ); + editor.assertProblems( + "bogus|Unknown", + "foo|Unused" + ); + } + + @Test + public void mergedAnchorNodesReconciled() throws Exception { + Editor editor = harness.newEditor( + "repo-dflts: &repo-dflts\n" + + " bogus: bar\n" + + "resources:\n" + + "- name: foo\n" + + " type: git\n" + + " source:\n" + + " <<: *repo-dflts\n" + ); + editor.assertProblems( + "bogus|Unknown", + "foo|Unused" + ); + } + + @Test + public void anchorsAndReferenceSample_1() throws Exception { + Editor editor = harness.newEditor( + "pool-template: &pool-template\n" + + " uri: ((pool-git-backing-store-uri))\n" + + " branch: master\n" + + " pool: OVERRIDEME\n" + + " private_key: ((pool-git-backing-store-private-key))\n" + + "\n" + + "sleep: &sleep\n" + + " config:\n" + + " platform: linux\n" + + " image_resource:\n" + + " type: docker-image\n" + + " source:\n" + + " repository: alpine\n" + + " tag: latest\n" + + " run:\n" + + " path: sh\n" + + " args:\n" + + " - -exc\n" + + " - sleep 60\n" + + "\n" + + "##########\n" + + "\n" + + "resource_types:\n" + + "\n" + + "- name: pool\n" + + " type: docker-image\n" + + " source:\n" + + " repository: ((pool-resource-docker-repo))\n" + + " tag: ((pool-resource-tag))\n" + + "\n" + + "##########\n" + + "\n" + + "resources:\n" + + "\n" + + "- name: acquire-pool\n" + + " type: pool\n" + + " source:\n" + + " <<: *pool-template\n" + + " pool: acquire-pool\n" + + "\n" + + "- name: claim-pool\n" + + " type: pool\n" + + " source:\n" + + " <<: *pool-template\n" + + " pool: claim-pool\n" + + "\n" + + "##########\n" + + "\n" + + "jobs:\n" + + "\n" + + "- name: acquire-1\n" + + " plan:\n" + + " - put: acquire-pool\n" + + " params: {acquire: true}\n" + + " - task: sleep\n" + + " <<: *sleep\n" + + " ensure:\n" + + " put: acquire-pool\n" + + " params: {release: acquire-pool}\n" + + "\n" + + "- name: claim-1\n" + + " plan:\n" + + " - put: claim-pool\n" + + " params: {claim: slot-1}\n" + + " - task: sleep\n" + + " <<: *sleep\n" + + " ensure:\n" + + " put: claim-pool\n" + + " params: {release: claim-pool}\n" + ); + + System.out.println("============================"); + System.out.println(editor.getRawText()); + System.out.println("============================"); + editor.assertProblems(/*none*/); + } + + @Test + public void anchorsAndReferenceSample_2() throws Exception { + Editor editor = harness.newEditor( + "dcind: &dcind\n" + + " type: docker-image\n" + + " source:\n" + + " repository: kiwiops/stuff-mem-dcind\n" + + " tag: latest\n" + + "jobs:\n" + + "- name: job-well-done\n" + + " plan:\n" + + " - task: deploy-ssp-devint\n" + + " privileged: true\n" + + " config:\n" + + " platform: linux\n" + + " image_resource: \n" + + " <<: *dcind\n" + + " inputs:\n" + + " - name: kms\n" + + " run:\n" + + " path: ls\n" + + " args:\n" + + " - '-la'" + ); + editor.assertProblems(/*None*/); + } + + @Test + public void anchorsAndReferenceSample_3() throws Exception { + Editor editor = harness.newEditor( + "resources:\n" + + " - name: hello_hapi\n" + + " type: git\n" + + " source: &repo-source\n" + + " uri: https://somewhere.com/your_github_user/hello_hapi.git\n" + + " branch: master\n" + + " - name: dependency-cache\n" + + " type: npm-cache\n" + + " source:\n" + + " <<: *repo-source\n" + + " paths:\n" + + " - package.json\n" + ); + editor.assertProblems( + "hello_hapi|Unused", + "dependency-cache|Unused", + "npm-cache|not exist" + ); + } + + @Test + public void anchorsAndReferenceSample_4() throws Exception { + Editor editor = harness.newEditor( + "resource_types:\n" + + " - name: npm-cache\n" + + " type: docker-image\n" + + " source:\n" + + " repository: ymedlop/npm-cache-resource\n" + + " tag: latest\n" + + "\n" + + "resources:\n" + + " - name: hello_hapi\n" + + " type: git\n" + + " source: &repo-source\n" + + " uri: https://somehost.com/your_github_user/hello_hapi.git\n" + + " branch: master\n" + + " - name: dependency-cache\n" + + " type: npm-cache\n" + + " source:\n" + + " <<: *repo-source\n" + + " paths:\n" + + " - package.json\n" + + "\n" + + "jobs:\n" + + " - name: Install dependencies\n" + + " plan:\n" + + " - get: hello_hapi\n" + + " trigger: true\n" + + " - get: dependency-cache\n" + + " - name: Run tests\n" + + " plan:\n" + + " - get: hello_hapi\n" + + " trigger: true\n" + + " passed: [Install dependencies]\n" + + " - get: dependency-cache\n" + + " passed: [Install dependencies]\n" + + " - task: run the test suite\n" + + " file: hello_hapi/ci/tasks/run_tests.yml\n" + ); + editor.assertProblems(/*None*/); + } + ////////////////////////////////////////////////////////////////////////////// private void assertContextualCompletions(String conText, String textBefore, String... textAfter) throws Exception {