Spring/spring-tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reconciler support for yaml anchors, references and merges

Browse Source
This commit is contained in:
Kris De Volder
2019-02-05 14:22:46 -08:00
parent df3b168396
commit 7d47662094
7 changed files with 427 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
headless-services/commons/commons-yaml/pom.xml
View File

@@ -26,7 +26,6 @@
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>${yaml-version}</version>
</dependency>
<dependency>
<groupId>javax.inject</groupId>

50
headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/AstDumper.java Normal file
View File

@@ -0,0 +1,50 @@
/*******************************************************************************
* Copyright (c) 2019 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Pivotal, Inc. - initial API and implementation
*******************************************************************************/
package org.springframework.ide.vscode.commons.yaml.ast;
import org.yaml.snakeyaml.nodes.MappingNode;
import org.yaml.snakeyaml.nodes.Node;
import org.yaml.snakeyaml.nodes.NodeTuple;
import org.yaml.snakeyaml.nodes.ScalarNode;
import org.yaml.snakeyaml.nodes.SequenceNode;
public class AstDumper {
public static void dump(Node node, int indent) {
if (node instanceof MappingNode) {
for (NodeTuple entry : ((MappingNode)node).getValue()) {
println(indent, NodeUtil.asScalar(entry.getKeyNode())+":");
dump(entry.getValueNode(), indent+1);
}
} else if (node instanceof SequenceNode) {
for (Node el : ((SequenceNode)node).getValue()) {
println(indent, "[");
dump(el, indent+1);
println(indent, "]");
}
} else if (node instanceof ScalarNode) {
println(indent, NodeUtil.asScalar(node));
} else {
println(indent, "???"+node.getClass().getSimpleName());
}
}
private static void println(int indent, String string) {
indent(indent);
System.out.println(string);
}
private static void indent(int indent) {
for (int i = 0; i < indent; i++) {
System.out.print(" ");
}
}
}

130
headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/ast/NodeMergeSupport.java Normal file
View File

@@ -0,0 +1,130 @@
/**
* Copyright (c) 2008, 2019 http://www.snakeyaml.org and others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Contributors:
*
* 2008 - http://www.snakeyaml.org original api and implementation.
* 2019 - Copied and modified by Pivotal.
*/
package org.springframework.ide.vscode.commons.yaml.ast;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.springframework.ide.vscode.commons.languageserver.reconcile.IProblemCollector;
import org.springframework.ide.vscode.commons.yaml.reconcile.YamlSchemaProblems;
import org.yaml.snakeyaml.nodes.MappingNode;
import org.yaml.snakeyaml.nodes.Node;
import org.yaml.snakeyaml.nodes.NodeTuple;
import org.yaml.snakeyaml.nodes.SequenceNode;
import org.yaml.snakeyaml.nodes.Tag;
/**
* Bits and pieces copied from snakeyaml library to support dealing with '<<' merge
* nodes.
* <p>
* Some modifications made to support our own use-cases.
*/
public class NodeMergeSupport {
private IProblemCollector problems;
public NodeMergeSupport(IProblemCollector problems) {
this.problems = problems;
}
public void flattenMapping(MappingNode node) {
// perform merging only on nodes containing merge node(s)
//processDuplicateKeys(node);
if (node.isMerged()) {
node.setValue(mergeNode(node, true, new HashMap<Object, Integer>(),
new ArrayList<NodeTuple>()));
node.setMerged(false);
}
}
/**
* Does merge for supplied mapping node.
*
* @param node
* where to merge
* @param isPreffered
* true if keys of node should take precedence over others...
* @param key2index
* maps already merged keys to index from values
* @param values
* collects merged NodeTuple
* @return list of the merged NodeTuple (to be set as value for the
* MappingNode)
*/
private List<NodeTuple> mergeNode(MappingNode node, boolean isPreffered,
Map<Object, Integer> key2index, List<NodeTuple> values) {
Iterator<NodeTuple> iter = node.getValue().iterator();
while (iter.hasNext()) {
final NodeTuple nodeTuple = iter.next();
final Node keyNode = nodeTuple.getKeyNode();
final Node valueNode = nodeTuple.getValueNode();
if (keyNode.getTag().equals(Tag.MERGE)) {
iter.remove();
switch (valueNode.getNodeId()) {
case mapping:
MappingNode mn = (MappingNode) valueNode;
mergeNode(mn, false, key2index, values);
break;
case sequence:
SequenceNode sn = (SequenceNode) valueNode;
List<Node> vals = sn.getValue();
for (Node subnode : vals) {
if (!(subnode instanceof MappingNode)) {
problems.accept(YamlSchemaProblems.schemaProblem(
"Expected a mapping for merging, but found "+subnode.getNodeId(), subnode
));
} else {
MappingNode mnode = (MappingNode) subnode;
mergeNode(mnode, false, key2index, values);
}
}
break;
default:
problems.accept(YamlSchemaProblems.schemaProblem(
"Expected a mapping or list of mappings for merging, but found "
+ valueNode.getNodeId(),
valueNode
));
}
} else {
// we need to construct keys to avoid duplications
String key = NodeUtil.asScalar(keyNode);
if (key!=null) {
if (!key2index.containsKey(key)) { // 1st time merging key
values.add(nodeTuple);
// keep track where tuple for the key is
key2index.put(key, values.size() - 1);
} else if (isPreffered) { // there is value for the key, but we
// need to override it
// change value for the key using saved position
values.set(key2index.get(key), nodeTuple);
}
}
}
}
return values;
}
}

31
headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/SchemaBasedYamlASTReconciler.java
View File

@@ -40,6 +40,7 @@ import org.springframework.ide.vscode.commons.util.ValueParseException;
import org.springframework.ide.vscode.commons.util.ValueParser;
import org.springframework.ide.vscode.commons.util.text.DocumentRegion;
import org.springframework.ide.vscode.commons.util.text.IDocument;
import org.springframework.ide.vscode.commons.yaml.ast.NodeMergeSupport;
import org.springframework.ide.vscode.commons.yaml.ast.NodeUtil;
import org.springframework.ide.vscode.commons.yaml.ast.YamlFileAST;
import org.springframework.ide.vscode.commons.yaml.path.YamlPath;
@@ -69,6 +70,7 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler {
private final YTypeUtil typeUtil;
private final ITypeCollector typeCollector;
private final YamlQuickfixes quickfixes;
private final NodeMergeSupport nodeMerger;
private List<Runnable> delayedConstraints = new ArrayList<>();
// keeps track of dynamic constraints discovered during reconciler walk
@@ -84,6 +86,7 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler {
this.typeCollector = typeCollector;
this.typeUtil = schema.getTypeUtil();
this.quickfixes = quickfixes;
this.nodeMerger = new NodeMergeSupport(problems);
}
@Override
@@ -138,7 +141,6 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler {
}
private void reconcile(YamlFileAST ast, YamlPath path, Node parent, Node node, YType _type) {
// IDocument doc = ast.getDocument();
if (_type!=null && !skipReconciling(node)) {
DynamicSchemaContext schemaContext = new ASTDynamicSchemaContext(ast, path, node);
YType type = typeUtil.inferMoreSpecificType(_type, schemaContext);
@@ -149,6 +151,7 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler {
switch (getNodeId(node)) {
case mapping:
MappingNode map = (MappingNode) node;
nodeMerger.flattenMapping(map);
checkForDuplicateKeys(map);
if (typeUtil.isMap(type)) {
for (NodeTuple entry : map.getValue()) {
@@ -167,7 +170,9 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler {
} else {
YTypedProperty prop = beanProperties.get(key);
if (prop==null) {
unknownBeanProperty(keyNode, type, key);
if (!isAnchored(entry)) {
unknownBeanProperty(keyNode, type, key);
}
} else {
if (prop.isDeprecated()) {
String msg = prop.getDeprecationMessage();
@@ -223,6 +228,28 @@ public class SchemaBasedYamlASTReconciler implements YamlASTReconciler {
}
}
/**
* Detects whether a given map key-value pair is anchored. I.e. corresponds to
* a bit of yaml like this example:
*
* <pre>
* some-key: &some-anchor
* blah: blah
* more: blah
* </pre>
*
* @param entry
* @return
*/
private boolean isAnchored(NodeTuple entry) {
if (entry!=null) {
Node v = entry.getValueNode();
String a = v.getAnchor();
return a!=null;
}
return false;
}
private void parse(YamlFileAST ast, Node node, YType type, ValueParser parser) {
try {
String value = NodeUtil.asScalar(node);

2
headless-services/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/reconcile/YamlSchemaProblems.java
View File

@@ -23,7 +23,6 @@ import org.springframework.ide.vscode.commons.languageserver.reconcile.ProblemTy
import org.springframework.ide.vscode.commons.languageserver.reconcile.ReconcileProblem;
import org.springframework.ide.vscode.commons.languageserver.reconcile.ReconcileProblemImpl;
import org.springframework.ide.vscode.commons.util.CollectionUtil;
import org.springframework.ide.vscode.commons.util.CollectorUtil;
import org.springframework.ide.vscode.commons.util.text.DocumentRegion;
import org.springframework.ide.vscode.commons.util.text.IDocument;
import org.springframework.ide.vscode.commons.yaml.ast.NodeUtil;
@@ -32,7 +31,6 @@ import org.springframework.ide.vscode.commons.yaml.path.YamlPathSegment;
import org.springframework.ide.vscode.commons.yaml.schema.DynamicSchemaContext;
import org.springframework.ide.vscode.commons.yaml.schema.YType;
import org.springframework.ide.vscode.commons.yaml.schema.YTypedProperty;
import org.springframework.ide.vscode.commons.yaml.snippet.Snippet;
import org.yaml.snakeyaml.error.Mark;
import org.yaml.snakeyaml.nodes.MappingNode;
import org.yaml.snakeyaml.nodes.Node;

1
headless-services/commons/pom.xml
View File

@@ -79,7 +79,6 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<yaml-version>1.17</yaml-version>
<junit-version>4.12</junit-version>
<assertj-version>3.5.2</assertj-version>
<slf4j-version>1.7.25</slf4j-version>

218
headless-services/concourse-language-server/src/test/java/org/springframework/ide/vscode/concourse/ConcourseEditorTest.java
View File

@@ -732,6 +732,7 @@ public class ConcourseEditorTest {
);
}
@Test
public void reconcileDuplicateResourceTypeNames_exempt_Builtins() throws Exception {
//See https://github.com/spring-projects/sts4/issues/196
@@ -4882,6 +4883,223 @@ public class ConcourseEditorTest {
editor.assertHoverContains("optional", "If `true`, then the input is not required by the task");
}
@Test
public void anchorNodeSuppressesUnknownPropertyError() throws Exception {
Editor editor = harness.newEditor(
"pool-template: &pool-template\n" +
" uri: ((pool-git-backing-store-uri))\n" +
" branch: master\n" +
" pool: OVERRIDEME\n" +
" private_key: ((pool-git-backing-store-private-key))\n"
);
editor.assertProblems(/*none*/);
}
@Test
public void referencedAnchorNodesReconciled() throws Exception {
Editor editor = harness.newEditor(
"repo-dflts: &repo-dflts\n" +
" bogus: bar\n" +
"resources:\n" +
"- name: foo\n" +
" type: git\n" +
" source: *repo-dflts\n"
);
editor.assertProblems(
"bogus|Unknown",
"foo|Unused"
);
}
@Test
public void mergedAnchorNodesReconciled() throws Exception {
Editor editor = harness.newEditor(
"repo-dflts: &repo-dflts\n" +
" bogus: bar\n" +
"resources:\n" +
"- name: foo\n" +
" type: git\n" +
" source:\n" +
" <<: *repo-dflts\n"
);
editor.assertProblems(
"bogus|Unknown",
"foo|Unused"
);
}
@Test
public void anchorsAndReferenceSample_1() throws Exception {
Editor editor = harness.newEditor(
"pool-template: &pool-template\n" +
" uri: ((pool-git-backing-store-uri))\n" +
" branch: master\n" +
" pool: OVERRIDEME\n" +
" private_key: ((pool-git-backing-store-private-key))\n" +
"\n" +
"sleep: &sleep\n" +
" config:\n" +
" platform: linux\n" +
" image_resource:\n" +
" type: docker-image\n" +
" source:\n" +
" repository: alpine\n" +
" tag: latest\n" +
" run:\n" +
" path: sh\n" +
" args:\n" +
" - -exc\n" +
" - sleep 60\n" +
"\n" +
"##########\n" +
"\n" +
"resource_types:\n" +
"\n" +
"- name: pool\n" +
" type: docker-image\n" +
" source:\n" +
" repository: ((pool-resource-docker-repo))\n" +
" tag: ((pool-resource-tag))\n" +
"\n" +
"##########\n" +
"\n" +
"resources:\n" +
"\n" +
"- name: acquire-pool\n" +
" type: pool\n" +
" source:\n" +
" <<: *pool-template\n" +
" pool: acquire-pool\n" +
"\n" +
"- name: claim-pool\n" +
" type: pool\n" +
" source:\n" +
" <<: *pool-template\n" +
" pool: claim-pool\n" +
"\n" +
"##########\n" +
"\n" +
"jobs:\n" +
"\n" +
"- name: acquire-1\n" +
" plan:\n" +
" - put: acquire-pool\n" +
" params: {acquire: true}\n" +
" - task: sleep\n" +
" <<: *sleep\n" +
" ensure:\n" +
" put: acquire-pool\n" +
" params: {release: acquire-pool}\n" +
"\n" +
"- name: claim-1\n" +
" plan:\n" +
" - put: claim-pool\n" +
" params: {claim: slot-1}\n" +
" - task: sleep\n" +
" <<: *sleep\n" +
" ensure:\n" +
" put: claim-pool\n" +
" params: {release: claim-pool}\n"
);
System.out.println("============================");
System.out.println(editor.getRawText());
System.out.println("============================");
editor.assertProblems(/*none*/);
}
@Test
public void anchorsAndReferenceSample_2() throws Exception {
Editor editor = harness.newEditor(
"dcind: &dcind\n" +
" type: docker-image\n" +
" source:\n" +
" repository: kiwiops/stuff-mem-dcind\n" +
" tag: latest\n" +
"jobs:\n" +
"- name: job-well-done\n" +
" plan:\n" +
" - task: deploy-ssp-devint\n" +
" privileged: true\n" +
" config:\n" +
" platform: linux\n" +
" image_resource: \n" +
" <<: *dcind\n" +
" inputs:\n" +
" - name: kms\n" +
" run:\n" +
" path: ls\n" +
" args:\n" +
" - '-la'"
);
editor.assertProblems(/*None*/);
}
@Test
public void anchorsAndReferenceSample_3() throws Exception {
Editor editor = harness.newEditor(
"resources:\n" +
" - name: hello_hapi\n" +
" type: git\n" +
" source: &repo-source\n" +
" uri: https://somewhere.com/your_github_user/hello_hapi.git\n" +
" branch: master\n" +
" - name: dependency-cache\n" +
" type: npm-cache\n" +
" source:\n" +
" <<: *repo-source\n" +
" paths:\n" +
" - package.json\n"
);
editor.assertProblems(
"hello_hapi|Unused",
"dependency-cache|Unused",
"npm-cache|not exist"
);
}
@Test
public void anchorsAndReferenceSample_4() throws Exception {
Editor editor = harness.newEditor(
"resource_types:\n" +
" - name: npm-cache\n" +
" type: docker-image\n" +
" source:\n" +
" repository: ymedlop/npm-cache-resource\n" +
" tag: latest\n" +
"\n" +
"resources:\n" +
" - name: hello_hapi\n" +
" type: git\n" +
" source: &repo-source\n" +
" uri: https://somehost.com/your_github_user/hello_hapi.git\n" +
" branch: master\n" +
" - name: dependency-cache\n" +
" type: npm-cache\n" +
" source:\n" +
" <<: *repo-source\n" +
" paths:\n" +
" - package.json\n" +
"\n" +
"jobs:\n" +
" - name: Install dependencies\n" +
" plan:\n" +
" - get: hello_hapi\n" +
" trigger: true\n" +
" - get: dependency-cache\n" +
" - name: Run tests\n" +
" plan:\n" +
" - get: hello_hapi\n" +
" trigger: true\n" +
" passed: [Install dependencies]\n" +
" - get: dependency-cache\n" +
" passed: [Install dependencies]\n" +
" - task: run the test suite\n" +
" file: hello_hapi/ci/tasks/run_tests.yml\n"
);
editor.assertProblems(/*None*/);
}
//////////////////////////////////////////////////////////////////////////////
private void assertContextualCompletions(String conText, String textBefore, String... textAfter) throws Exception {