Add support for s3 resource type
This commit is contained in:
Kris De Volder
@@ -13,6 +13,7 @@ package org.springframework.ide.vscode.concourse;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.ide.vscode.commons.util.MimeTypes;
|
||||
import org.springframework.ide.vscode.commons.util.Renderable;
|
||||
import org.springframework.ide.vscode.commons.util.Renderables;
|
||||
import org.springframework.ide.vscode.commons.util.ValueParsers;
|
||||
@@ -94,6 +95,9 @@ public class PipelineYmlSchema implements YamlSchema {
|
||||
public final YAtomicType t_resource_name;
|
||||
public final YAtomicType t_job_name;
|
||||
public final YAtomicType t_resource_type_name;
|
||||
public final YType t_mime_type = f.yatomic("MimeType")
|
||||
.parseWith(ValueParsers.NE_STRING)
|
||||
.addHints(MimeTypes.getKnownMimeTypes());
|
||||
|
||||
public final YBeanType task;
|
||||
|
||||
@@ -361,6 +365,53 @@ public class PipelineYmlSchema implements YamlSchema {
|
||||
|
||||
resourceTypes.def("docker-image", source, get, put);
|
||||
}
|
||||
//s3
|
||||
{
|
||||
String[] validRegions = {
|
||||
};
|
||||
YType t_s3_region = f.yenum("S3Region",
|
||||
//See: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html
|
||||
"us-west-1", "us-west-2",
|
||||
"ca-central-1", "EU", "eu-west-1",
|
||||
"eu-west-2", "eu-central-1",
|
||||
"ap-south-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2",
|
||||
"sa-east-1",
|
||||
"us-east-2"
|
||||
);
|
||||
|
||||
YType t_canned_acl = f.yenum("S3CannedAcl",
|
||||
//See http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
"private", "public-read", "public-read-write", "aws-exec-read",
|
||||
"authenticated-read", "bucket-owner-read", "bucket-owner-full-control",
|
||||
"log-delivery-write"
|
||||
);
|
||||
|
||||
YBeanType source = f.ybean("S3Source");
|
||||
addProp(source, "bucket", t_ne_string).isRequired(true);
|
||||
addProp(source, "access_key_id", t_ne_string);
|
||||
addProp(source, "secret_access_key", t_ne_string);
|
||||
addProp(source, "region_name", t_s3_region);
|
||||
addProp(source, "private", t_boolean);
|
||||
addProp(source, "cloudfront_url", t_ne_string);
|
||||
addProp(source, "endpoint", t_ne_string);
|
||||
addProp(source, "disable_ssl", t_boolean);
|
||||
addProp(source, "server_side_encryption", t_ne_string);
|
||||
addProp(source, "sse_kms_key_id", t_ne_string);
|
||||
addProp(source, "use_v2_signing", t_boolean);
|
||||
addProp(source, "regexp", t_ne_string);
|
||||
addProp(source, "versioned_file", t_ne_string);
|
||||
source.requireOneOf("regexp", "versioned_file");
|
||||
|
||||
YBeanType get = f.ybean("S3GetParams");
|
||||
//Note: S3GetParams intentionally has no properties since no params are expected according to the docs.
|
||||
|
||||
YBeanType put = f.ybean("S3PutParams");
|
||||
addProp(put, "file", t_ne_string).isRequired(true);
|
||||
addProp(put, "acl", t_canned_acl);
|
||||
addProp(put, "content_type", t_mime_type);
|
||||
|
||||
resourceTypes.def("s3", source, get, put);
|
||||
}
|
||||
}
|
||||
|
||||
private String getResourceType(String resourceNameProp, ConcourseModel models, DynamicSchemaContext dc) {
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
*Optional.* [Canned Acl](http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html)
|
||||
for the uploaded object.
|
||||
@@ -0,0 +1,2 @@
|
||||
*Optional.* MIME [Content-Type](https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17)
|
||||
describing the contents of the uploaded object
|
||||
@@ -0,0 +1,5 @@
|
||||
*Required.* Path to the file to upload, provided by an output of a task.
|
||||
If multiple files are matched by the glob, an error is raised. The file which
|
||||
matches will be placed into the directory structure on S3 as defined in `regexp`
|
||||
in the resource definition. The matching syntax is bash glob expansion, so
|
||||
no capture groups, etc.
|
||||
@@ -0,0 +1 @@
|
||||
*Optional.* The AWS access key to use when accessing the bucket.
|
||||
@@ -0,0 +1 @@
|
||||
*Required.* The name of the bucket.
|
||||
@@ -0,0 +1,6 @@
|
||||
*Optional.* The URL (scheme and domain) of your CloudFront distribution that is fronting this bucket (e.g
|
||||
`https://d5yxxxxx.cloudfront.net`). This will affect `in` but not `check`
|
||||
and `put`. `in` will ignore the `bucket` name setting, exclusively using the
|
||||
`cloudfront_url`. When configuring CloudFront with versioned buckets, set
|
||||
`Query String Forwarding and Caching` to `Forward all, cache based on all` to
|
||||
ensure S3 calls succeed.
|
||||
@@ -0,0 +1,2 @@
|
||||
*Optional.* Disable SSL for the endpoint, useful for S3
|
||||
compatible providers without SSL.
|
||||
@@ -0,0 +1 @@
|
||||
*Optional.* Custom endpoint for using S3 compatible provider.
|
||||
@@ -0,0 +1,2 @@
|
||||
*Optional.* Indicates that the bucket is private, so that any
|
||||
URLs provided are signed.
|
||||
@@ -0,0 +1,4 @@
|
||||
*Optional.* The pattern to match filenames against within S3. The first
|
||||
grouped match is used to extract the version, or if a group is explicitly
|
||||
named `version`, that group is used. At least one capture group must be
|
||||
specified, with parentheses.
|
||||
@@ -0,0 +1 @@
|
||||
*Optional.* The region the bucket is in. Defaults to `us-east-1`.
|
||||
@@ -0,0 +1 @@
|
||||
*Optional.* The AWS secret key to use when accessing the bucket.
|
||||
@@ -0,0 +1 @@
|
||||
*Optional.* An encryption algorithm to use when storing objects in S3.
|
||||
@@ -0,0 +1,2 @@
|
||||
*Optional.* The ID of the AWS KMS master encryption key
|
||||
used for the object.
|
||||
@@ -0,0 +1 @@
|
||||
*Optional.* Use signature v2 signing, useful for S3 compatible providers that do not support v4.
|
||||
@@ -0,0 +1,4 @@
|
||||
*Optional* If you enable versioning for your S3 bucket then
|
||||
you can keep the file name the same and upload new versions of your file
|
||||
without resorting to version numbers. This property is the path to the file
|
||||
in your S3 bucket.
|
||||
@@ -1304,6 +1304,192 @@ public class ConcourseEditorTest {
|
||||
editor.assertHoverContains("skip_download", "Skip `docker pull`");
|
||||
}
|
||||
|
||||
@Test public void s3ResourceSourceReconcileAndHovers() throws Exception {
|
||||
Editor editor;
|
||||
|
||||
editor = harness.newEditor(
|
||||
"resources:\n" +
|
||||
"- name: s3-snapshots\n" +
|
||||
" type: s3\n" +
|
||||
" source:\n" +
|
||||
" access_key_id: the-key"
|
||||
);
|
||||
editor.assertProblems("access_key_id: the-key|'bucket' is required");
|
||||
|
||||
editor = harness.newEditor(
|
||||
"resources:\n" +
|
||||
"- name: s3-snapshots\n" +
|
||||
" type: s3\n" +
|
||||
" source:\n" +
|
||||
" bucket: the-bucket\n" +
|
||||
" access_key_id: the-access-key\n" +
|
||||
" secret_access_key: the-secret-key\n" +
|
||||
" region_name: bogus-region\n" +
|
||||
" private: is-private\n" +
|
||||
" cloudfront_url: https://d5yxxxxx.cloudfront.net\n" +
|
||||
" endpoint: https://blah.custom.com/blah/blah\n" +
|
||||
" disable_ssl: no_ssl_checking\n" +
|
||||
" server_side_encryption: some-encryption-algo\n" + //TODO: validation and CA? What values are acceptable?
|
||||
" sse_kms_key_id: the-master-key-id\n" +
|
||||
" use_v2_signing: should-use-v2\n" +
|
||||
" regexp: path-to-file-(.*).tar.gz\n" +
|
||||
" versioned_file: path/to/file.tar.gz\n"
|
||||
);
|
||||
editor.assertProblems(
|
||||
"bogus-region|unknown 'S3Region'",
|
||||
"is-private|'boolean'",
|
||||
"no_ssl_checking|'boolean'",
|
||||
"should-use-v2|'boolean'"
|
||||
);
|
||||
|
||||
editor.assertHoverContains("bucket", "The name of the bucket");
|
||||
editor.assertHoverContains("access_key_id", "The AWS access key");
|
||||
editor.assertHoverContains("secret_access_key", "The AWS secret key");
|
||||
editor.assertHoverContains("region_name", "The region the bucket is in");
|
||||
editor.assertHoverContains("private", "Indicates that the bucket is private");
|
||||
editor.assertHoverContains("cloudfront_url", "The URL (scheme and domain) of your CloudFront distribution");
|
||||
editor.assertHoverContains("endpoint", "Custom endpoint for using S3");
|
||||
editor.assertHoverContains("disable_ssl", "Disable SSL for the endpoint");
|
||||
editor.assertHoverContains("server_side_encryption", "An encryption algorithm to use");
|
||||
editor.assertHoverContains("sse_kms_key_id", "The ID of the AWS KMS master encryption key");
|
||||
editor.assertHoverContains("use_v2_signing", "Use signature v2 signing");
|
||||
editor.assertHoverContains("regexp", "The pattern to match filenames against within S3");
|
||||
editor.assertHoverContains("versioned_file", "If you enable versioning for your S3 bucket");
|
||||
}
|
||||
|
||||
@Test public void s3ResourceRegionCompletions() throws Exception {
|
||||
String[] validRegions = {
|
||||
//See: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html
|
||||
"us-west-1", "us-west-2", "ca-central-1",
|
||||
"EU", "eu-west-1", "eu-west-2", "eu-central-1",
|
||||
"ap-south-1", "ap-southeast-1", "ap-southeast-2",
|
||||
"ap-northeast-1", "ap-northeast-2",
|
||||
"sa-east-1", "us-east-2"
|
||||
};
|
||||
Arrays.sort(validRegions);
|
||||
|
||||
String[] expectedCompletions = new String[validRegions.length];
|
||||
for (int i = 0; i < expectedCompletions.length; i++) {
|
||||
expectedCompletions[i] = validRegions[i]+"<*>";
|
||||
}
|
||||
|
||||
assertContextualCompletions(
|
||||
"resources:\n" +
|
||||
"- name: s3-snapshots\n" +
|
||||
" type: s3\n" +
|
||||
" source:\n" +
|
||||
" region_name: <*>"
|
||||
, //===================
|
||||
"<*>"
|
||||
, // ===>
|
||||
expectedCompletions
|
||||
);
|
||||
}
|
||||
|
||||
@Test public void s3ResourceGetParamsReconcileAndHovers() throws Exception {
|
||||
Editor editor;
|
||||
|
||||
editor = harness.newEditor(
|
||||
"resources:\n" +
|
||||
"- name: my-s3-bucket\n" +
|
||||
" type: s3\n" +
|
||||
"jobs:\n" +
|
||||
"- name: a-job\n" +
|
||||
" plan:\n" +
|
||||
" - get: my-s3-bucket\n" +
|
||||
" params:\n" +
|
||||
" no-params-expected: bad"
|
||||
);
|
||||
|
||||
editor.assertProblems(
|
||||
"no-params-expected|Unknown property"
|
||||
);
|
||||
}
|
||||
|
||||
@Test public void s3ResourcePutParamsReconcileAndHovers() throws Exception {
|
||||
Editor editor;
|
||||
|
||||
editor = harness.newEditor(
|
||||
"resources:\n" +
|
||||
"- name: my-s3-bucket\n" +
|
||||
" type: s3\n" +
|
||||
"jobs:\n" +
|
||||
"- name: a-job\n" +
|
||||
" plan:\n" +
|
||||
" - put: my-s3-bucket\n" +
|
||||
" params:\n" +
|
||||
" acl: public-read\n" +
|
||||
" get_params:\n" +
|
||||
" no-params-expected: bad"
|
||||
);
|
||||
editor.assertProblems(
|
||||
"acl: public-read|'file' is required",
|
||||
"no-params-expected|Unknown property"
|
||||
);
|
||||
|
||||
editor = harness.newEditor(
|
||||
"resources:\n" +
|
||||
"- name: my-s3-bucket\n" +
|
||||
" type: s3\n" +
|
||||
"jobs:\n" +
|
||||
"- name: a-job\n" +
|
||||
" plan:\n" +
|
||||
" - put: my-s3-bucket\n" +
|
||||
" params:\n" +
|
||||
" file: path/to/file\n" +
|
||||
" acl: bad-acl\n" +
|
||||
" content_type: anything/goes\n"
|
||||
);
|
||||
editor.assertProblems(
|
||||
"bad-acl|unknown 'S3CannedAcl'"
|
||||
);
|
||||
|
||||
editor.assertHoverContains("file", "Path to the file to upload");
|
||||
editor.assertHoverContains("acl", "Canned Acl");
|
||||
editor.assertHoverContains("content_type", "MIME");
|
||||
}
|
||||
|
||||
@Test public void s3ResourcePutParamsContentAssist() throws Exception {
|
||||
String[] cannedAcls = { //See http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
"private",
|
||||
"public-read",
|
||||
"public-read-write",
|
||||
"aws-exec-read",
|
||||
"authenticated-read",
|
||||
"bucket-owner-read",
|
||||
"bucket-owner-full-control",
|
||||
"log-delivery-write"
|
||||
};
|
||||
Arrays.sort(cannedAcls);
|
||||
|
||||
String conText =
|
||||
"resources:\n" +
|
||||
"- name: my-s3-bucket\n" +
|
||||
" type: s3\n" +
|
||||
"jobs:\n" +
|
||||
"- name: a-job\n" +
|
||||
" plan:\n" +
|
||||
" - put: my-s3-bucket\n" +
|
||||
" params:\n" +
|
||||
" <*>";
|
||||
|
||||
assertContextualCompletions(conText,
|
||||
"content_type: json<*>"
|
||||
, //=>
|
||||
"content_type: application/json; charset=utf-8<*>"
|
||||
);
|
||||
|
||||
String[] expectedAclCompletions = new String[cannedAcls.length];
|
||||
for (int i = 0; i < expectedAclCompletions.length; i++) {
|
||||
expectedAclCompletions[i] = "acl: "+cannedAcls[i]+"<*>";
|
||||
}
|
||||
assertContextualCompletions(conText,
|
||||
"acl: <*>"
|
||||
, // ===>
|
||||
expectedAclCompletions
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void gotoResourceDefinition() throws Exception {
|
||||
Editor editor = harness.newEditor(
|
||||
|
||||
Reference in New Issue
Block a user