Add new attributes to S3 resource schema

GetParams:
- skip_download
- unpack

Source:
- skip_download
- skip_ssl_verification
- session_token
This commit is contained in:
Kris De Volder
2018-10-29 11:54:17 -07:00
parent 83ded61abb
commit 05fbcd81f4
7 changed files with 43 additions and 4 deletions

View File

@@ -543,11 +543,14 @@ public class PipelineYmlSchema implements YamlSchema {
addProp(source, "bucket", t_ne_string).isPrimary(true);
addProp(source, "access_key_id", t_ne_string);
addProp(source, "secret_access_key", t_ne_string);
addProp(source, "session_token", t_ne_string);
addProp(source, "region_name", t_s3_region);
addProp(source, "private", t_boolean);
addProp(source, "cloudfront_url", t_ne_string);
addProp(source, "endpoint", t_ne_string);
addProp(source, "disable_ssl", t_boolean);
addProp(source, "skip_ssl_verification", t_boolean);
addProp(source, "skip_download", t_boolean);
addProp(source, "server_side_encryption", t_ne_string);
addProp(source, "sse_kms_key_id", t_ne_string);
addProp(source, "use_v2_signing", t_boolean);
@@ -556,7 +559,8 @@ public class PipelineYmlSchema implements YamlSchema {
source.requireOneOf("regexp", "versioned_file");
AbstractType get = f.ybean("S3GetParams");
//Note: S3GetParams intentionally has no properties since no params are expected according to the docs.
addProp(get, "unpack", t_boolean);
addProp(get, "skip_download", t_boolean);
AbstractType put = f.ybean("S3PutParams");
addProp(put, "file", t_ne_string).isPrimary(true);

View File

@@ -0,0 +1 @@
*Optional*. Skip downloading object from S3. Same parameter as source configuration but used to define/override by get.

View File

@@ -0,0 +1,2 @@
*Optional*. If true and the file is an archive (tar, gzipped tar, other gzipped file, or zip), unpack the file.
Gzipped tarballs will be both ungzipped and untarred. It is ignored when `get` is running on the initial version.

View File

@@ -0,0 +1 @@
*Optional*. The AWS STS session token to use when accessing the bucket.

View File

@@ -0,0 +1 @@
*Optional*. Skip downloading object from S3. Useful to only trigger the pipeline without using the object.

View File

@@ -0,0 +1 @@
*Optional*. Skip SSL verification for S3 endpoint. Useful for S3 compatible providers using self-signed SSL certificates.