Merge pull request #778 from spring-projects/LukeBalizet-update-registry-image-schema

Luke balizet update registry image schema

headless-services/concourse-language-server/src/main/java/org/springframework/ide/vscode/concourse/PipelineYmlSchema.java

@@ -62,6 +62,7 @@ import com.google.common.collect.ImmutableList;
 
 /**
  * @author Kris De Volder
+ * @author LukeBalizet
  */
 public class PipelineYmlSchema implements YamlSchema {
 
@@ -156,14 +157,32 @@ public class PipelineYmlSchema implements YamlSchema {
 	public final YType t_semver = f.yatomic("Semver")
 			.parseWith(ValueParsers.NE_STRING); //TODO: use real semver parser.
 
-	public final YType t_s3_region = f.yenum("S3Region",
+	public final YType t_aws_region = f.yenum("AWSRegion",
-			//See: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html
+			//See https://docs.aws.amazon.com/general/latest/gr/rande.html
-			 "us-west-1", "us-west-2",
+			"af-south-1",
-			 "ca-central-1", "EU", "eu-west-1",
+			"ap-east-1",
-			 "eu-west-2", "eu-central-1",
+			"ap-southeast-3",
-			 "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2",
+			"ap-south-1",
-			 "sa-east-1",
+			"ap-northeast-3",
-			 "us-east-2"
+			"ap-northeast-2",
+			"ap-southeast-1",
+			"ap-southeast-2",
+			"ap-northeast-1",
+			"ca-central-1",
+			"cn-north-1",
+			"cn-northwest-1",
+			"eu-central-1",
+			"eu-west-1",
+			"eu-west-2",
+			"eu-south-1",
+			"eu-west-3",
+			"eu-north-1",
+			"me-south-1",
+			"us-east-1",
+			"us-east-2",
+			"us-west-1",
+			"us-west-2",
+			"sa-east-1"
 	);
 
 	public final YType t_day = f.yenum("Day",
@@ -606,14 +625,16 @@ public class PipelineYmlSchema implements YamlSchema {
 			put.require(Constraints.requireAtMostOneOf("rebase", "merge"));
 			resourceTypes.def("git", source, get, put);
 		}
+
+		// Shared by docker-image and registry-image
+		AbstractType registry_mirror = f.ybean("RegistryMirror");
+		addProp(registry_mirror, "host", t_ne_string).isPrimary(true);
+		addProp(registry_mirror, "username", t_ne_string);
+		addProp(registry_mirror, "password", t_ne_string);
+		
 		//docker-image:
 		{
 			
-			AbstractType registry_mirror = f.ybean("RegistryMirror");
-			addProp(registry_mirror, "host", t_ne_string).isPrimary(true);
-			addProp(registry_mirror, "username", t_ne_string);
-			addProp(registry_mirror, "password", t_ne_string);
-			
 			AbstractType source = f.ybean("DockerImageSource");
 			addProp(source, "repository", t_ne_string).isPrimary(true);
 			addProp(source, "tag", t_ne_string);
@@ -670,14 +691,24 @@ public class PipelineYmlSchema implements YamlSchema {
 
 			resourceTypes.def("docker-image", source, get, put);
 		}
-		//registry_image
+		//registry-image
 		{
 			AbstractType source = f.ybean("RegistryImageSource");
 			addProp(source, "repository", t_ne_string).isPrimary(true);
+			addProp(source, "insecure", t_boolean);
 			addProp(source, "tag", t_ne_string);
+			addProp(source, "variant", t_ne_string);
+			addProp(source, "semver_constraint", t_ne_string);
 			addProp(source, "username", t_ne_string);
 			addProp(source, "password", t_ne_string);
+			addProp(source, "aws_access_key_id", t_ne_string);
+			addProp(source, "aws_secret_access_key", t_ne_string);
+			addProp(source, "aws_session_token", t_ne_string);
+			addProp(source, "aws_region", t_aws_region);
+			addProp(source, "aws_role_arn", t_ne_string);
+			addProp(source, "aws_role_arns", t_strings);
 			addProp(source, "debug", t_boolean);
+			addProp(source, "registry_mirror", registry_mirror);
 			{
 				AbstractType contentTrust = f.ybean("RegistryImageContentTrust");
 				addProp(contentTrust, "server", t_ne_string);
@@ -689,6 +720,7 @@ public class PipelineYmlSchema implements YamlSchema {
 				
 				addProp(source, "content_trust", contentTrust);
 			}
+			addProp(source, "ca_certs", t_strings);
 			
 			AbstractType get = f.ybean("RegistryImageGetParams");
 			addProp(get, "format", f.yenum("RegistryImageFormat", "rootfs", "oci"));
@@ -696,9 +728,13 @@ public class PipelineYmlSchema implements YamlSchema {
 			
 			AbstractType put = f.ybean("RegistryImagePutParams");
 			addProp(put, "image", t_ne_string).isPrimary(true);
+			addProp(put, "version", t_ne_string);
+			addProp(put, "bump_aliases", t_boolean);
 			addProp(put, "additional_tags", t_ne_string);
 
 			resourceTypes.def("registry-image", source, get, put);
+			
+			source.require(Constraints.mutuallyExclusive("aws_role_arn", "aws_role_arns"));
 		}
 		//s3
 		{
@@ -714,7 +750,7 @@ public class PipelineYmlSchema implements YamlSchema {
 			addProp(source, "access_key_id", t_ne_string);
 			addProp(source, "secret_access_key", t_ne_string);
 			addProp(source, "session_token", t_ne_string);
-			addProp(source, "region_name", t_s3_region);
+			addProp(source, "region_name", t_aws_region);
 			addProp(source, "private", t_boolean);
 			addProp(source, "cloudfront_url", t_ne_string);
 			addProp(source, "endpoint", t_ne_string);
@@ -790,7 +826,7 @@ public class PipelineYmlSchema implements YamlSchema {
 			addProp(s3_source, "key", t_ne_string).isRequired(true);
 			addProp(s3_source, "access_key_id", t_ne_string).isRequired(true);
 			addProp(s3_source, "secret_access_key", t_ne_string).isRequired(true);
-			addProp(s3_source, "region_name", t_s3_region);
+			addProp(s3_source, "region_name", t_aws_region);
 			addProp(s3_source, "endpoint", t_ne_string);
 			addProp(s3_source, "disable_ssl", t_boolean);
 

headless-services/concourse-language-server/src/main/resources/desc/RegistryImagePutParams/bump_aliases.md

@@ -0,0 +1,11 @@
+*Optional. Default `false`.* When set to `true` and `version` is specified
+automatically bump alias tags for the version. For example, when pushing version
+`1.2.3`, push the same image to the following tags:
+- `1.2`, if 1.2.3 is the latest version of 1.2.x.
+- `1`, if 1.2.3 is the latest version of 1.x.
+- `latest`, if 1.2.3 is the latest version overall.
+
+If `variant` is configured as `foo`, push the same image to the following tags:
+- `1.2-foo`, if 1.2.3 is the latest version of 1.2.x with `foo`.
+- `1-foo`, if 1.2.3 is the latest version of 1.x with `foo`.
+- `foo`, if 1.2.3 is the latest version overall for `foo`

headless-services/concourse-language-server/src/main/resources/desc/RegistryImagePutParams/version.md

@@ -0,0 +1 @@
+*Optional.* A version number to use as a tag.

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/aws_access_key_id.md

@@ -0,0 +1 @@
+*Optional*. The access key ID to use for authenticating with ECR.

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/aws_region.md

@@ -0,0 +1,4 @@
+*Optional*. The region to use for
+  accessing ECR. This is required if you are using ECR. This region
+  will help determine the full repository URL you are accessing
+  (e.g., `012345678910.dkr.ecr.us-east-1.amazonaws.com`)

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/aws_role_arn.md

@@ -0,0 +1,3 @@
+*Optional*. If set, then this role will
+  be assumed before authenticating to ECR. An error will occur if
+  `aws_role_arns` is also specified. This is kept for backward compatibility.

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/aws_role_arns.md

@@ -0,0 +1,4 @@
+*Optional*. An array of AWS IAM roles.
+  If set, these roles will be assumed in the specified order before
+  authenticating to ECR. An error will occur if `aws_role_arn`
+  is also specified.

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/aws_secret_access_key.md

@@ -0,0 +1 @@
+*Optional*. The secret access key to use for authenticating with ECR.

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/aws_session_token.md

@@ -0,0 +1,2 @@
+*Optional*. The session token to use for authenticating with
+  STS credentials with ECR.

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/ca_certs.md

@@ -0,0 +1,15 @@
+*Optional*. An array of PEM-encoded CA certificates. Example:
+```yaml
+ca_certs:
+- |
+  -----BEGIN CERTIFICATE-----
+  ...
+  -----END CERTIFICATE-----
+- |
+  -----BEGIN CERTIFICATE-----
+  ...
+  -----END CERTIFICATE-----
+``` 
+Each entry specifies the x509 CA certificate for the trusted docker registry.
+This is used to validate the certificate of the docker registry when the
+registry's certificate is signed by a custom authority (or itself).

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/insecure.md

@@ -0,0 +1 @@
+*Optional. Default `false`.* Allow insecure registry.

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/registry_mirror.md

@@ -0,0 +1 @@
+*Optional.* Hostname and credentials pointing to a docker registry mirror service.

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/semver_constraint.md

@@ -0,0 +1,7 @@
+*Optional.* Constrain the returned semver
+  tags according to a semver constraint, e.g.
+  `"~1.2.x"`, `">= 1.2 < 3.0.0 || >= 4.2.3"`. Follows the rules outlined in
+  [https://github.com/Masterminds/semver#checking-version-constraints](https://github.com/Masterminds/semver#checking-version-constraints)
+
+
+  

headless-services/concourse-language-server/src/main/resources/desc/RegistryImageSource/variant.md

@@ -0,0 +1,6 @@
+*Optional.* Detect only tags matching this
+  variant suffix, and push version tags with this suffix applied. For example,
+  a value of `stretch` would be used for tags like `1.2.3-stretch`. This is
+  typically used *without* `tag` - if it is set, this value will only used for
+  pushing, not checking.
+  

headless-services/concourse-language-server/src/test/java/org/springframework/ide/vscode/concourse/ConcourseEditorTest.java

@@ -2603,30 +2603,6 @@ public class ConcourseEditorTest {
 			"my-docker-image|Unused 'Resource'",
 			"source|'repository' is required"
 		);
-
-		editor = harness.newEditor(
-				"resources:\n" +
-				"- name: my-docker-image\n" +
-				"  type: registry-image\n" +
-				"  source:\n" +
-				"    repository: kdvolder/sts4-build-env\n" +
-				"    tag: latest\n" +
-				"    username: kdvolder\n" +
-				"    password: {{docker_password}}\n" +
-				"    debug: no-bool\n" +
-				"    content_trust: {}\n"
-		);
-		editor.assertProblems(
-				"my-docker-image|Unused 'Resource'",
-				"no-bool|boolean",
-				"content_trust|Properties [repository_key, repository_key_id, repository_passphrase] are required"
-		);
-
-		editor.assertHoverContains("repository", "The name of the repository");
-		editor.assertHoverContains("tag", "name of the tag");
-		editor.assertHoverContains("username", "username to use");
-		editor.assertHoverContains("password", "password to use");
-		editor.assertHoverContains("debug",  "debugging output will be printed");
 		
 		editor = harness.newEditor(
 				"resources:\n" +
@@ -2634,6 +2610,68 @@ public class ConcourseEditorTest {
 				"  type: registry-image\n" +
 				"  source:\n" +
 				"    repository: kdvolder/sts4-build-env\n" +
+				"    insecure: this-is-not-safe\n" +
+				"    tag: latest\n" +
+				"    variant: some-suffix\n" +
+				"    semver_constraint: invalid-semver-exp\n" +
+				"    username: kdvolder\n" +
+				"    password: {{docker_password}}\n" +
+				"    aws_access_key_id: the-key-to-aws\n" +
+				"    aws_secret_access_key: the-aws-secret\n" +
+				"    aws_session_token: aws-is-in-session\n" +
+				"    aws_region: bad-aws-region\n" +
+				"    aws_role_arn: some-arn\n" +
+				"    aws_role_arns: a-list-of-arns\n" +
+				"    debug: no-bool\n" +
+				"    registry_mirror: {}\n" +
+				"    content_trust: {}\n" +
+				"    ca_certs: some-certs\n"
+		);
+		
+		editor.assertProblems(
+				"my-docker-image|Unused 'Resource'",
+				"this-is-not-safe|boolean",
+				//"invalid-semver-exp|Invalid semver constraint", //TODO: parse / check this?
+				"bad-aws-region|AWSRegion",
+				"aws_role_arn|Only one of 'aws_role_arn' and 'aws_role_arns'",
+				"aws_role_arns|Only one of 'aws_role_arn' and 'aws_role_arns'",
+				"a-list-of-arns|Expecting a 'Sequence'",
+				"no-bool|boolean",
+				"registry_mirror|'host' is required",
+				"content_trust|Properties [repository_key, repository_key_id, repository_passphrase] are required",
+				"some-certs|Expecting a 'Sequence'"
+		);
+
+		editor.assertHoverContains("repository", "The name of the repository");
+		editor.assertHoverContains("insecure", "Allow insecure registry");
+		editor.assertHoverContains("tag", "name of the tag");
+		editor.assertHoverContains("variant", "variant suffix");
+		editor.assertHoverContains("semver_constraint", "Constrain the returned semver");
+		editor.assertHoverContains("username", "username to use");
+		editor.assertHoverContains("password", "password to use");
+		editor.assertHoverContains("aws_access_key_id", "access key ID to use for authenticating with ECR");
+		editor.assertHoverContains("aws_secret_access_key", "secret access key to use for authenticating with ECR");
+		editor.assertHoverContains("aws_session_token", "session token to use");
+		editor.assertHoverContains("aws_region", "region to use");
+		editor.assertHoverContains("aws_role_arn", "this role will");
+		editor.assertHoverContains("aws_role_arns", "assumed in the specified order");
+		editor.assertHoverContains("debug",  "debugging output will be printed");
+		editor.assertHoverContains("registry_mirror", "pointing to a docker registry mirror service");
+		
+		editor = harness.newEditor(
+				"resources:\n" +
+				"- name: my-docker-image\n" +
+				"  type: registry-image\n" +
+				"  source:\n" +
+				"    repository: kdvolder/sts4-build-env\n" +
+				"    ca_certs:\n" +
+				"    - cert1\n" +
+				"    - cert2\n" +
+				"    registry_mirror:\n" +
+				"      host: mirrorhost\n" +
+				"      username: mirroruser\n" +
+				"      password: mirrorpass\n" +
+				"      not_expected_in_mirror: bad\n" +
 				"    content_trust:\n"+
 				"      server: notary.server\n" +
 				"      repository_key: repokey\n" +
@@ -2645,8 +2683,12 @@ public class ConcourseEditorTest {
 		);
 		editor.assertProblems(
 				"my-docker-image|Unused 'Resource'",
+				"not_expected_in_mirror|Unknown property",
 				"bogus_prop|Unknown property"
 		);
+		editor.assertHoverContains("host", "hostname pointing to a Docker registry");
+		editor.assertHoverContains("username", "username to use");
+		editor.assertHoverContains("password", "password to use");
 		editor.assertHoverContains("server", "URL for the notary server");
 		editor.assertHoverContains("repository_key_id", "ID used to sign the trusted collection");
 		editor.assertHoverContains("repository_key", "Target key used to sign");
@@ -2684,7 +2726,11 @@ public class ConcourseEditorTest {
 
 	@Test public void registryImageResourcePutParamsReconcileAndHovers() throws Exception {
 		Editor editor;
-
+		
+		//TODO: new properties 
+		// version
+		// bump_aliases
+		
 		editor = harness.newEditor(
 				"resources:\n" +
 				"- name: my-docker-image\n" +
@@ -2695,6 +2741,8 @@ public class ConcourseEditorTest {
 				"  - put: my-docker-image\n" +
 				"    params:\n" +
 				"      image: path/to/image/tarball\n" +
+				"      version: some-version\n" +
+				"      bump_aliases: is-bump\n" +
 				"      additional_tags: path/to/tagsfiles\n" +
 				"      bogus: bad\n" +
 				"    get_params:\n" +
@@ -2702,11 +2750,14 @@ public class ConcourseEditorTest {
 		);
 
 		editor.assertProblems(
+				"is-bump|boolean",
 				"bogus|Unknown property",
 				"bad-format|Valid values are: [oci, rootfs]"
 		);
 
 		editor.assertHoverContains("image", 4, "path to the OCI image tarball");
+		editor.assertHoverContains("version", "version number to use as a tag");
+		editor.assertHoverContains("bump_aliases", "automatically bump alias tags");
 		editor.assertHoverContains("additional_tags", "list of tag values");
 		editor.assertHoverContains("format", "The format to fetch as");
 	}
@@ -2794,7 +2845,7 @@ public class ConcourseEditorTest {
 		);
 		editor.assertProblems(
 				"s3-snapshots|Unused 'Resource'",
-				"bogus-region|unknown 'S3Region'",
+				"bogus-region|unknown 'AWSRegion'",
 				"is-private|'boolean'",
 				"no_ssl_checking|'boolean'",
 				"skipping-ssl|'boolean'",
@@ -2834,12 +2885,30 @@ public class ConcourseEditorTest {
 
 	@Test public void s3ResourceRegionCompletions() throws Exception {
 		String[] validRegions = {
-				//See: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html
+				"af-south-1",
-				"us-west-1", "us-west-2", "ca-central-1",
+				"ap-east-1",
-				"EU", "eu-west-1", "eu-west-2", "eu-central-1",
+				"ap-southeast-3",
-				"ap-south-1", "ap-southeast-1", "ap-southeast-2",
+				"ap-south-1",
-				"ap-northeast-1", "ap-northeast-2",
+				"ap-northeast-3",
-				"sa-east-1", "us-east-2"
+				"ap-northeast-2",
+				"ap-southeast-1",
+				"ap-southeast-2",
+				"ap-northeast-1",
+				"ca-central-1",
+				"cn-north-1",
+				"cn-northwest-1",
+				"eu-central-1",
+				"eu-west-1",
+				"eu-west-2",
+				"eu-south-1",
+				"eu-west-3",
+				"eu-north-1",
+				"me-south-1",
+				"us-east-1",
+				"us-east-2",
+				"us-west-1",
+				"us-west-2",
+				"sa-east-1"
 		};
 		Arrays.sort(validRegions);
 
@@ -3489,7 +3558,7 @@ public class ConcourseEditorTest {
 		);
 		editor.assertProblems(
 				"version|Unused 'Resource'",
-				"bogus-region|'S3Region'",
+				"bogus-region|'AWSRegion'",
 				"no-use-ssl|'boolean'",
 				"bogus-prop|Unknown property"
 		);
@@ -3514,7 +3583,7 @@ public class ConcourseEditorTest {
 		);
 		editor.assertProblems(
 				"version|Unused 'Resource'",
-				"bogus-region|'S3Region'",
+				"bogus-region|'AWSRegion'",
 				"no-use-ssl|'boolean'",
 				"bogus-prop|Unknown property"
 		);