Spring/spring-tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for s3 resource type

Browse Source
This commit is contained in:
Kris De Volder
2017-01-28 14:49:34 -08:00
parent 8695e66626
commit 928817a6b5
22 changed files with 327 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
vscode-extensions/commons/commons-util/src/main/java/org/springframework/ide/vscode/commons/util/MimeTypes.java Normal file
View File

@@ -0,0 +1,38 @@
/*******************************************************************************
* Copyright (c) 2017 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Pivotal, Inc. - initial API and implementation
*******************************************************************************/
package org.springframework.ide.vscode.commons.util;
import java.lang.reflect.Field;
import java.util.Map;
import java.util.TreeSet;
import com.google.common.net.MediaType;
public class MimeTypes {
public static String[] getKnownMimeTypes() {
try {
Field f = MediaType.class.getDeclaredField("KNOWN_TYPES");
f.setAccessible(true);
@SuppressWarnings("unchecked")
Map<MediaType, MediaType> map = (Map<MediaType, MediaType>) f.get(null);
TreeSet<String> mediaTypes = new TreeSet<>();
for (MediaType m : map.keySet()) {
mediaTypes.add(m.toString());
}
return mediaTypes.toArray(new String[mediaTypes.size()]);
} catch (Exception e) {
Log.log(e);
}
return null;
}
}

5
vscode-extensions/commons/commons-yaml/src/main/java/org/springframework/ide/vscode/commons/yaml/schema/YTypeFactory.java
View File

@@ -418,6 +418,11 @@ public class YTypeFactory {
return true;
}
public void requireOneOf(String... properties) {
Assert.isLegal(properties.length>1);
//TODO: implement support for this.
}
}
public static class YAtomicType extends AbstractType {

26
vscode-extensions/vscode-boot-properties/src/main/java/org/springframework/ide/vscode/boot/metadata/types/TypeUtil.java
View File

@@ -14,7 +14,6 @@ package org.springframework.ide.vscode.boot.metadata.types;
import static org.springframework.ide.vscode.commons.util.ArrayUtils.firstElement;
import static org.springframework.ide.vscode.commons.util.ArrayUtils.lastElement;
import java.lang.reflect.Field;
import java.net.InetAddress;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
@@ -30,7 +29,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -55,12 +53,12 @@ import org.springframework.ide.vscode.commons.util.EnumValueParser;
import org.springframework.ide.vscode.commons.util.HtmlSnippet;
import org.springframework.ide.vscode.commons.util.LazyProvider;
import org.springframework.ide.vscode.commons.util.Log;
import org.springframework.ide.vscode.commons.util.MimeTypes;
import org.springframework.ide.vscode.commons.util.StringUtil;
import org.springframework.ide.vscode.commons.util.ValueParser;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.MediaType;
import reactor.core.publisher.Flux;
@@ -185,36 +183,43 @@ public class TypeUtil {
private static final Map<String,ValueParser> VALUE_PARSERS = new HashMap<>();
static {
VALUE_PARSERS.put(Byte.class.getName(), new RadixableParser() {
@Override
public Object parse(String str, int radix) {
return Byte.parseByte(str, radix);
}
});
VALUE_PARSERS.put(Integer.class.getName(), new RadixableParser() {
@Override
public Object parse(String str, int radix) {
return Integer.parseInt(str, radix);
}
});
VALUE_PARSERS.put(Long.class.getName(), new RadixableParser() {
@Override
public Object parse(String str, int radix) {
return Long.parseLong(str, radix);
}
});
VALUE_PARSERS.put(Short.class.getName(), new RadixableParser() {
@Override
public Object parse(String str, int radix) {
return Short.parseShort(str, radix);
}
});
VALUE_PARSERS.put(Double.class.getName(), new ValueParser() {
@Override
public Object parse(String str) {
return Double.parseDouble(str);
}
});
VALUE_PARSERS.put(Float.class.getName(), new ValueParser() {
@Override
public Object parse(String str) {
return Float.parseFloat(str);
}
});
VALUE_PARSERS.put(Boolean.class.getName(), new ValueParser() {
@Override
public Object parse(String str) {
//The 'more obvious' implementation is too liberal and accepts anything as okay.
//return Boolean.parseBoolean(str);
@@ -573,20 +578,7 @@ public class TypeUtil {
valueHints("org.springframework.util.MimeType", new LazyProvider<String[]>() {
@Override
protected String[] compute() {
try {
Field f = MediaType.class.getDeclaredField("KNOWN_TYPES");
f.setAccessible(true);
@SuppressWarnings("unchecked")
Map<MediaType, MediaType> map = (Map<MediaType, MediaType>) f.get(null);
TreeSet<String> mediaTypes = new TreeSet<>();
for (MediaType m : map.keySet()) {
mediaTypes.add(m.toString());
}
return mediaTypes.toArray(new String[mediaTypes.size()]);
} catch (Exception e) {
Log.log(e);
}
return null;
return MimeTypes.getKnownMimeTypes();
}
});
valueHints("org.springframework.core.io.Resource", new ResourceHintProvider());

3
vscode-extensions/vscode-concourse/README.md
View File

@@ -48,8 +48,9 @@ The resource-types that are already defined in the schema are:
- git
- docker-image
- s3
For other resource-types content assist and checking is still very limited. However, we intend
For other resource-types content assist and checking is still very limited. We intend
to grow this list and provide a similar level of support for all of the built-in resource types in
the near future.

51
vscode-extensions/vscode-concourse/src/main/java/org/springframework/ide/vscode/concourse/PipelineYmlSchema.java
View File

@@ -13,6 +13,7 @@ package org.springframework.ide.vscode.concourse;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.ide.vscode.commons.util.MimeTypes;
import org.springframework.ide.vscode.commons.util.Renderable;
import org.springframework.ide.vscode.commons.util.Renderables;
import org.springframework.ide.vscode.commons.util.ValueParsers;
@@ -94,6 +95,9 @@ public class PipelineYmlSchema implements YamlSchema {
public final YAtomicType t_resource_name;
public final YAtomicType t_job_name;
public final YAtomicType t_resource_type_name;
public final YType t_mime_type = f.yatomic("MimeType")
.parseWith(ValueParsers.NE_STRING)
.addHints(MimeTypes.getKnownMimeTypes());
public final YBeanType task;
@@ -361,6 +365,53 @@ public class PipelineYmlSchema implements YamlSchema {
resourceTypes.def("docker-image", source, get, put);
}
//s3
{
String[] validRegions = {
};
YType t_s3_region = f.yenum("S3Region",
//See: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html
"us-west-1", "us-west-2",
"ca-central-1", "EU", "eu-west-1",
"eu-west-2", "eu-central-1",
"ap-south-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2",
"sa-east-1",
"us-east-2"
);
YType t_canned_acl = f.yenum("S3CannedAcl",
//See http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
"private", "public-read", "public-read-write", "aws-exec-read",
"authenticated-read", "bucket-owner-read", "bucket-owner-full-control",
"log-delivery-write"
);
YBeanType source = f.ybean("S3Source");
addProp(source, "bucket", t_ne_string).isRequired(true);
addProp(source, "access_key_id", t_ne_string);
addProp(source, "secret_access_key", t_ne_string);
addProp(source, "region_name", t_s3_region);
addProp(source, "private", t_boolean);
addProp(source, "cloudfront_url", t_ne_string);
addProp(source, "endpoint", t_ne_string);
addProp(source, "disable_ssl", t_boolean);
addProp(source, "server_side_encryption", t_ne_string);
addProp(source, "sse_kms_key_id", t_ne_string);
addProp(source, "use_v2_signing", t_boolean);
addProp(source, "regexp", t_ne_string);
addProp(source, "versioned_file", t_ne_string);
source.requireOneOf("regexp", "versioned_file");
YBeanType get = f.ybean("S3GetParams");
//Note: S3GetParams intentionally has no properties since no params are expected according to the docs.
YBeanType put = f.ybean("S3PutParams");
addProp(put, "file", t_ne_string).isRequired(true);
addProp(put, "acl", t_canned_acl);
addProp(put, "content_type", t_mime_type);
resourceTypes.def("s3", source, get, put);
}
}
private String getResourceType(String resourceNameProp, ConcourseModel models, DynamicSchemaContext dc) {

2
vscode-extensions/vscode-concourse/src/main/resources/desc/S3PutParams/acl.md Normal file
View File

@@ -0,0 +1,2 @@
*Optional.* [Canned Acl](http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html)
for the uploaded object.

2
vscode-extensions/vscode-concourse/src/main/resources/desc/S3PutParams/content_type.md Normal file
View File

@@ -0,0 +1,2 @@
*Optional.* MIME [Content-Type](https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17)
describing the contents of the uploaded object

5
vscode-extensions/vscode-concourse/src/main/resources/desc/S3PutParams/file.md Normal file
View File

@@ -0,0 +1,5 @@
*Required.* Path to the file to upload, provided by an output of a task.
If multiple files are matched by the glob, an error is raised. The file which
matches will be placed into the directory structure on S3 as defined in `regexp`
in the resource definition. The matching syntax is bash glob expansion, so
no capture groups, etc.

1
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/access_key_id.md Normal file
View File

@@ -0,0 +1 @@
*Optional.* The AWS access key to use when accessing the bucket.

1
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/bucket.md Normal file
View File

@@ -0,0 +1 @@
*Required.* The name of the bucket.

6
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/cloudfront_url.md Normal file
View File

@@ -0,0 +1,6 @@
*Optional.* The URL (scheme and domain) of your CloudFront distribution that is fronting this bucket (e.g
`https://d5yxxxxx.cloudfront.net`). This will affect `in` but not `check`
and `put`. `in` will ignore the `bucket` name setting, exclusively using the
`cloudfront_url`. When configuring CloudFront with versioned buckets, set
`Query String Forwarding and Caching` to `Forward all, cache based on all` to
ensure S3 calls succeed.

2
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/disable_ssl.md Normal file
View File

@@ -0,0 +1,2 @@
*Optional.* Disable SSL for the endpoint, useful for S3
compatible providers without SSL.

1
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/endpoint.md Normal file
View File

@@ -0,0 +1 @@
*Optional.* Custom endpoint for using S3 compatible provider.

2
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/private.md Normal file
View File

@@ -0,0 +1,2 @@
*Optional.* Indicates that the bucket is private, so that any
URLs provided are signed.

4
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/regexp.md Normal file
View File

@@ -0,0 +1,4 @@
*Optional.* The pattern to match filenames against within S3. The first
grouped match is used to extract the version, or if a group is explicitly
named `version`, that group is used. At least one capture group must be
specified, with parentheses.

1
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/region_name.md Normal file
View File

@@ -0,0 +1 @@
*Optional.* The region the bucket is in. Defaults to `us-east-1`.

1
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/secret_access_key.md Normal file
View File

@@ -0,0 +1 @@
*Optional.* The AWS secret key to use when accessing the bucket.

1
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/server_side_encryption.md Normal file
View File

@@ -0,0 +1 @@
*Optional.* An encryption algorithm to use when storing objects in S3.

2
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/sse_kms_key_id.md Normal file
View File

@@ -0,0 +1,2 @@
*Optional.* The ID of the AWS KMS master encryption key
used for the object.

1
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/use_v2_signing.md Normal file
View File

@@ -0,0 +1 @@
*Optional.* Use signature v2 signing, useful for S3 compatible providers that do not support v4.

4
vscode-extensions/vscode-concourse/src/main/resources/desc/S3Source/versioned_file.md Normal file
View File

@@ -0,0 +1,4 @@
*Optional* If you enable versioning for your S3 bucket then
you can keep the file name the same and upload new versions of your file
without resorting to version numbers. This property is the path to the file
in your S3 bucket.

186
vscode-extensions/vscode-concourse/src/test/java/org/springframework/ide/vscode/concourse/ConcourseEditorTest.java
View File

@@ -1304,6 +1304,192 @@ public class ConcourseEditorTest {
editor.assertHoverContains("skip_download", "Skip `docker pull`");
}
@Test public void s3ResourceSourceReconcileAndHovers() throws Exception {
Editor editor;
editor = harness.newEditor(
"resources:\n" +
"- name: s3-snapshots\n" +
" type: s3\n" +
" source:\n" +
" access_key_id: the-key"
);
editor.assertProblems("access_key_id: the-key|'bucket' is required");
editor = harness.newEditor(
"resources:\n" +
"- name: s3-snapshots\n" +
" type: s3\n" +
" source:\n" +
" bucket: the-bucket\n" +
" access_key_id: the-access-key\n" +
" secret_access_key: the-secret-key\n" +
" region_name: bogus-region\n" +
" private: is-private\n" +
" cloudfront_url: https://d5yxxxxx.cloudfront.net\n" +
" endpoint: https://blah.custom.com/blah/blah\n" +
" disable_ssl: no_ssl_checking\n" +
" server_side_encryption: some-encryption-algo\n" + //TODO: validation and CA? What values are acceptable?
" sse_kms_key_id: the-master-key-id\n" +
" use_v2_signing: should-use-v2\n" +
" regexp: path-to-file-(.*).tar.gz\n" +
" versioned_file: path/to/file.tar.gz\n"
);
editor.assertProblems(
"bogus-region|unknown 'S3Region'",
"is-private|'boolean'",
"no_ssl_checking|'boolean'",
"should-use-v2|'boolean'"
);
editor.assertHoverContains("bucket", "The name of the bucket");
editor.assertHoverContains("access_key_id", "The AWS access key");
editor.assertHoverContains("secret_access_key", "The AWS secret key");
editor.assertHoverContains("region_name", "The region the bucket is in");
editor.assertHoverContains("private", "Indicates that the bucket is private");
editor.assertHoverContains("cloudfront_url", "The URL (scheme and domain) of your CloudFront distribution");
editor.assertHoverContains("endpoint", "Custom endpoint for using S3");
editor.assertHoverContains("disable_ssl", "Disable SSL for the endpoint");
editor.assertHoverContains("server_side_encryption", "An encryption algorithm to use");
editor.assertHoverContains("sse_kms_key_id", "The ID of the AWS KMS master encryption key");
editor.assertHoverContains("use_v2_signing", "Use signature v2 signing");
editor.assertHoverContains("regexp", "The pattern to match filenames against within S3");
editor.assertHoverContains("versioned_file", "If you enable versioning for your S3 bucket");
}
@Test public void s3ResourceRegionCompletions() throws Exception {
String[] validRegions = {
//See: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html
"us-west-1", "us-west-2", "ca-central-1",
"EU", "eu-west-1", "eu-west-2", "eu-central-1",
"ap-south-1", "ap-southeast-1", "ap-southeast-2",
"ap-northeast-1", "ap-northeast-2",
"sa-east-1", "us-east-2"
};
Arrays.sort(validRegions);
String[] expectedCompletions = new String[validRegions.length];
for (int i = 0; i < expectedCompletions.length; i++) {
expectedCompletions[i] = validRegions[i]+"<*>";
}
assertContextualCompletions(
"resources:\n" +
"- name: s3-snapshots\n" +
" type: s3\n" +
" source:\n" +
" region_name: <*>"
, //===================
"<*>"
, // ===>
expectedCompletions
);
}
@Test public void s3ResourceGetParamsReconcileAndHovers() throws Exception {
Editor editor;
editor = harness.newEditor(
"resources:\n" +
"- name: my-s3-bucket\n" +
" type: s3\n" +
"jobs:\n" +
"- name: a-job\n" +
" plan:\n" +
" - get: my-s3-bucket\n" +
" params:\n" +
" no-params-expected: bad"
);
editor.assertProblems(
"no-params-expected|Unknown property"
);
}
@Test public void s3ResourcePutParamsReconcileAndHovers() throws Exception {
Editor editor;
editor = harness.newEditor(
"resources:\n" +
"- name: my-s3-bucket\n" +
" type: s3\n" +
"jobs:\n" +
"- name: a-job\n" +
" plan:\n" +
" - put: my-s3-bucket\n" +
" params:\n" +
" acl: public-read\n" +
" get_params:\n" +
" no-params-expected: bad"
);
editor.assertProblems(
"acl: public-read|'file' is required",
"no-params-expected|Unknown property"
);
editor = harness.newEditor(
"resources:\n" +
"- name: my-s3-bucket\n" +
" type: s3\n" +
"jobs:\n" +
"- name: a-job\n" +
" plan:\n" +
" - put: my-s3-bucket\n" +
" params:\n" +
" file: path/to/file\n" +
" acl: bad-acl\n" +
" content_type: anything/goes\n"
);
editor.assertProblems(
"bad-acl|unknown 'S3CannedAcl'"
);
editor.assertHoverContains("file", "Path to the file to upload");
editor.assertHoverContains("acl", "Canned Acl");
editor.assertHoverContains("content_type", "MIME");
}
@Test public void s3ResourcePutParamsContentAssist() throws Exception {
String[] cannedAcls = { //See http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
"private",
"public-read",
"public-read-write",
"aws-exec-read",
"authenticated-read",
"bucket-owner-read",
"bucket-owner-full-control",
"log-delivery-write"
};
Arrays.sort(cannedAcls);
String conText =
"resources:\n" +
"- name: my-s3-bucket\n" +
" type: s3\n" +
"jobs:\n" +
"- name: a-job\n" +
" plan:\n" +
" - put: my-s3-bucket\n" +
" params:\n" +
" <*>";
assertContextualCompletions(conText,
"content_type: json<*>"
, //=>
"content_type: application/json; charset=utf-8<*>"
);
String[] expectedAclCompletions = new String[cannedAcls.length];
for (int i = 0; i < expectedAclCompletions.length; i++) {
expectedAclCompletions[i] = "acl: "+cannedAcls[i]+"<*>";
}
assertContextualCompletions(conText,
"acl: <*>"
, // ===>
expectedAclCompletions
);
}
@Test
public void gotoResourceDefinition() throws Exception {
Editor editor = harness.newEditor(