From e1f173e139eff8372505a39fbbabc54c8cf41793 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Tue, 23 Aug 2022 20:29:37 -0400 Subject: [PATCH] Schema Registry Migration Initial attempt at migrating schema registry from https://github.com/spring-cloud/spring-cloud-schema-registry to Spring Cloud Stream. Schema Registry within the scope of Spring Cloud Stream will only support Spring Cloud Stream specific use cases in the context of binders. Resolves https://github.com/spring-cloud/spring-cloud-stream/issues/2491 Resolves #2492 --- pom.xml | 1 + schema-registry/pom.xml | 56 ++ .../.jdk8 | 0 .../foodorder.avro | Bin 0 -> 274 bytes .../pom.xml | 119 ++++ .../stream/schema/registry/ParsedSchema.java | 65 ++ .../registry/SchemaNotFoundException.java | 28 + .../schema/registry/SchemaReference.java | 105 +++ .../registry/SchemaRegistrationResponse.java | 44 ++ .../avro/AbstractAvroMessageConverter.java | 142 ++++ ...AvroMessageConverterAutoConfiguration.java | 95 +++ .../avro/AvroMessageConverterProperties.java | 115 ++++ .../avro/AvroSchemaMessageConverter.java | 152 +++++ ...oSchemaRegistryClientMessageConverter.java | 413 ++++++++++++ .../avro/AvroSchemaServiceManager.java | 75 +++ .../avro/AvroSchemaServiceManagerImpl.java | 172 +++++ .../avro/DefaultSubjectNamingStrategy.java | 35 + .../avro/OriginalContentTypeResolver.java | 60 ++ .../avro/QualifiedSubjectNamingStrategy.java | 36 + .../registry/avro/SubjectNamingStrategy.java | 37 ++ .../avro/SubjectPrefixOnlyNamingStrategy.java | 31 + .../client/CachingRegistryClient.java | 67 ++ .../client/ConfluentSchemaRegistryClient.java | 166 +++++ .../client/DefaultSchemaRegistryClient.java | 104 +++ .../client/EnableSchemaRegistryClient.java | 41 ++ .../registry/client/SchemaRegistryClient.java | 54 ++ .../SchemaRegistryClientConfiguration.java | 55 ++ .../SchemaRegistryClientProperties.java | 49 ++ .../main/resources/META-INF/spring.factories | 3 + .../schema/avro/AvroSchemaLocationsTest.java | 122 ++++ .../avro/AvroSchemaMessageConverterTests.java | 165 +++++ .../avro/AvroSchemaServiceManagerTests.java | 190 ++++++ .../schema/avro/StubSchemaRegistryClient.java | 114 ++++ .../avro/SubjectNamingStrategyTest.java | 109 +++ .../cloud/stream/schema/avro/User1.java | 58 ++ .../cloud/stream/schema/avro/User2.java | 70 ++ .../ConfluentSchemaRegistryClientTests.java | 206 ++++++ .../stream/schema/avro/domain/FoodOrder.java | 44 ++ .../cloud/stream/schema/avro/v2/User1.java | 70 ++ ...vroMessageConverterSerializationTests.java | 234 +++++++ ...maRegistryClientMessageConverterTests.java | 214 ++++++ .../src/test/resources/schemas/Command.avsc | 19 + .../test/resources/schemas/imports/Email.avsc | 19 + .../schemas/imports/PushNotification.avsc | 15 + .../test/resources/schemas/imports/Sms.avsc | 14 + .../src/test/resources/schemas/status.avsc | 10 + .../src/test/resources/schemas/user.avsc | 10 + .../schemas/user1_multiple_records.schema | 22 + .../test/resources/schemas/user1_v1.schema | 10 + .../test/resources/schemas/user1_v2.schema | 10 + .../src/test/resources/schemas/user_v2.avsc | 10 + .../test/resources/schemas/users_v1.schema | 10 + .../test/resources/schemas/users_v2.schema | 10 + .../.jdk8 | 0 .../pom.xml | 41 ++ .../registry/EnableSchemaRegistryServer.java | 39 ++ .../config/SchemaServerConfiguration.java | 64 ++ .../config/SchemaServerProperties.java | 57 ++ .../controllers/ServerController.java | 274 ++++++++ .../schema/registry/model/Compatibility.java | 44 ++ .../stream/schema/registry/model/Schema.java | 93 +++ .../registry/repository/SchemaRepository.java | 36 + .../registry/support/AvroSchemaValidator.java | 83 +++ .../support/InvalidSchemaException.java | 28 + .../SchemaDeletionNotAllowedException.java | 32 + .../support/SchemaNotFoundException.java | 28 + .../registry/support/SchemaValidator.java | 70 ++ .../support/UnsupportedFormatException.java | 28 + .../src/main/resources/application.yml | 6 + .../AbstractServerControllerTest.java | 62 ++ .../entityScanning/EntityScanningTests.java | 46 ++ .../EntityScanningTestsWithEntityScan.java | 47 ++ .../entityScanning/ServerControllerTest.java | 65 ++ .../entityScanning/domain/TestEntity.java | 51 ++ .../.jdk8 | 0 .../pom.xml | 52 ++ .../SchemaRegistryServerApplication.java | 36 + .../main/resources/META-INF/spring.factories | 1 + .../src/main/resources/application.yml | 6 + .../server/SchemaRegistryServerAvroTests.java | 623 ++++++++++++++++++ .../avro_user_definition_schema_v1.json | 18 + .../avro_user_definition_schema_v2.json | 25 + .../src/test/resources/invalid_schema.json | 11 + 83 files changed, 6041 insertions(+) create mode 100644 schema-registry/pom.xml create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/.jdk8 create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/foodorder.avro create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/pom.xml create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/ParsedSchema.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaNotFoundException.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaReference.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaRegistrationResponse.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AbstractAvroMessageConverter.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroMessageConverterAutoConfiguration.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroMessageConverterProperties.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaMessageConverter.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaRegistryClientMessageConverter.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaServiceManager.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaServiceManagerImpl.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/DefaultSubjectNamingStrategy.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/OriginalContentTypeResolver.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/QualifiedSubjectNamingStrategy.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/SubjectNamingStrategy.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/SubjectPrefixOnlyNamingStrategy.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/CachingRegistryClient.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/ConfluentSchemaRegistryClient.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/DefaultSchemaRegistryClient.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/EnableSchemaRegistryClient.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/SchemaRegistryClient.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/config/SchemaRegistryClientConfiguration.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/config/SchemaRegistryClientProperties.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/main/resources/META-INF/spring.factories create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaLocationsTest.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaMessageConverterTests.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaServiceManagerTests.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/StubSchemaRegistryClient.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/SubjectNamingStrategyTest.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/User1.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/User2.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/client/ConfluentSchemaRegistryClientTests.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/domain/FoodOrder.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/v2/User1.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/serialization/AvroMessageConverterSerializationTests.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/serialization/AvroSchemaRegistryClientMessageConverterTests.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/Command.avsc create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/Email.avsc create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/PushNotification.avsc create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/Sms.avsc create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/status.avsc create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user.avsc create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_multiple_records.schema create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_v1.schema create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_v2.schema create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user_v2.avsc create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/users_v1.schema create mode 100644 schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/users_v2.schema create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/.jdk8 create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/pom.xml create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/EnableSchemaRegistryServer.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/config/SchemaServerConfiguration.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/config/SchemaServerProperties.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/controllers/ServerController.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/model/Compatibility.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/model/Schema.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/repository/SchemaRepository.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/AvroSchemaValidator.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/InvalidSchemaException.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaDeletionNotAllowedException.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaNotFoundException.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaValidator.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/UnsupportedFormatException.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/main/resources/application.yml create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/AbstractServerControllerTest.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/EntityScanningTests.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/EntityScanningTestsWithEntityScan.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/ServerControllerTest.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/domain/TestEntity.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/.jdk8 create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/pom.xml create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/src/main/java/org/springframework/cloud/stream/schema/registry/server/SchemaRegistryServerApplication.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/src/main/resources/META-INF/spring.factories create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/src/main/resources/application.yml create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/src/test/java/org/springframework/cloud/stream/schema/registry/server/SchemaRegistryServerAvroTests.java create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/avro_user_definition_schema_v1.json create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/avro_user_definition_schema_v2.json create mode 100644 schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/invalid_schema.json diff --git a/pom.xml b/pom.xml index ec2dfbf9d..b05eee0ca 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ core binders + schema-registry bom docs samples diff --git a/schema-registry/pom.xml b/schema-registry/pom.xml new file mode 100644 index 000000000..6a79f7f0e --- /dev/null +++ b/schema-registry/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + spring-cloud-stream-schema-registry + 4.0.0-SNAPSHOT + schema-registry + Spring Cloud Stream Schema Registry Components + pom + + + org.springframework.cloud + spring-cloud-stream-parent + 4.0.0-SNAPSHOT + + + + 1.9.2 + 1.4.192 + 2.13.2 + + + + spring-cloud-stream-schema-registry-core + spring-cloud-stream-schema-registry-server + spring-cloud-stream-schema-registry-client + + + + + + com.fasterxml.jackson + jackson-bom + ${jackson-bom.version} + import + pom + + + org.apache.avro + avro + ${avro.version} + + + com.h2database + h2 + ${h2.version} + + + org.springframework.cloud + spring-cloud-stream + ${project.version} + test + + + + + diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/.jdk8 b/schema-registry/spring-cloud-stream-schema-registry-client/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/foodorder.avro b/schema-registry/spring-cloud-stream-schema-registry-client/foodorder.avro new file mode 100644 index 0000000000000000000000000000000000000000..2607b1c54aca4f44bb8734644251c8cf1a2edd7d GIT binary patch literal 274 zcmaKmI}XAy5JdSj8~{;LxQ=`PIz9r4CM^=8F!ln5I2-LcAV9ejJy+le+<-%Xg8~Hw z<<8E$nRvOaH#%xhC|Pg7seAJ*{4oR369gK`h)AT$mG>CY#AJXfi8G~39ce8&Bb%;Q z6r2a7ozw~DMzSYRt|H5Ki$SMs8VD;3iDA*3pFey8_9`cp`tYtc1VZ47BKo>?&K@05 n-gbtz#E literal 0 HcmV?d00001 diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/pom.xml b/schema-registry/spring-cloud-stream-schema-registry-client/pom.xml new file mode 100644 index 000000000..bb8ef9064 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/pom.xml @@ -0,0 +1,119 @@ + + + + spring-cloud-stream-schema-registry + org.springframework.cloud + 4.0.0-SNAPSHOT + + 4.0.0 + + spring-cloud-stream-schema-registry-client + + + + org.springframework + spring-messaging + + + org.springframework.cloud + spring-cloud-stream + + + org.springframework + spring-web + + + org.springframework.boot + spring-boot-starter + + + com.fasterxml.jackson.core + jackson-databind + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.apache.avro + avro + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-stream + ${project.version} + test-jar + test + test-binder + + + org.springframework.cloud + spring-cloud-stream-schema-registry-core + ${project.version} + test + + + com.fasterxml.jackson.dataformat + jackson-dataformat-avro + + + org.apache.avro + avro + + + test + + + + + + + + + + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + generate-test-sources + + schema + + + + + ${project.basedir}/target/generated-test-sources + + + ${project.basedir}/target/generated-test-sources + + ${project.basedir}/src/test/resources/schemas + + + **/*.avsc + + + + ${project.basedir}/src/test/resources/schemas/imports/Email.avsc + + + ${project.basedir}/src/test/resources/schemas/imports/Sms.avsc + + + ${project.basedir}/src/test/resources/schemas/imports/PushNotification.avsc + + + + + + + diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/ParsedSchema.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/ParsedSchema.java new file mode 100644 index 000000000..9035cf763 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/ParsedSchema.java @@ -0,0 +1,65 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry; + +import org.apache.avro.Schema; + +/** + * Stores a {@link Schema} together with its String representation. + * + * Helps to avoid unnecessary parsing of schema textual representation, as well as calls + * to {@link org.apache.avro.Schema} toString method which is very expensive due the + * utilization of {@link com.fasterxml.jackson.databind.ObjectMapper} to output a JSON + * representation of the schema. + * + * Once a schema is found for any Class, be it a POJO or a + * {@link org.apache.avro.generic.GenericContainer}, both textual representation as well + * as the {@link org.apache.avro.Schema} will be stored within this class. + * + * @author Vinicius Carvalho + * + */ +public class ParsedSchema { + + private final Schema schema; + + private final String representation; + + private SchemaRegistrationResponse registration; + + public ParsedSchema(Schema schema) { + this.schema = schema; + this.representation = schema.toString(); + } + + public Schema getSchema() { + return this.schema; + } + + public String getRepresentation() { + return this.representation; + } + + public SchemaRegistrationResponse getRegistration() { + return this.registration; + } + + public void setRegistration(SchemaRegistrationResponse registration) { + this.registration = registration; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaNotFoundException.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaNotFoundException.java new file mode 100644 index 000000000..992ba9e58 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaNotFoundException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry; + +/** + * @author Vinicius Carvalho + */ +public class SchemaNotFoundException extends RuntimeException { + + public SchemaNotFoundException(String message) { + super(message); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaReference.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaReference.java new file mode 100644 index 000000000..060fdb039 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaReference.java @@ -0,0 +1,105 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry; + +import org.springframework.util.Assert; + +/** + * References a schema through its subject and version. + * + * @author Marius Bogoevici + */ +public class SchemaReference { + + private String subject; + + private int version; + + private String format; + + public SchemaReference(String subject, int version, String format) { + Assert.hasText(subject, "cannot be empty"); + Assert.isTrue(version > 0, "must be a positive integer"); + Assert.hasText(format, "cannot be empty"); + this.subject = subject; + this.version = version; + this.format = format; + } + + public String getSubject() { + return this.subject; + } + + public void setSubject(String subject) { + Assert.hasText(subject, "cannot be empty"); + this.subject = subject; + } + + public int getVersion() { + return this.version; + } + + public void setVersion(int version) { + Assert.isTrue(version > 0, "must be a positive integer"); + this.version = version; + } + + public String getFormat() { + return this.format; + } + + public void setFormat(String format) { + Assert.hasText(format, "cannot be empty"); + this.format = format; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + SchemaReference that = (SchemaReference) o; + + if (this.version != that.version) { + return false; + } + if (!this.subject.equals(that.subject)) { + return false; + } + return this.format.equals(that.format); + + } + + @Override + public int hashCode() { + int result = this.subject.hashCode(); + result = 31 * result + this.version; + result = 31 * result + this.format.hashCode(); + return result; + } + + @Override + public String toString() { + return "SchemaReference{" + "subject='" + this.subject + '\'' + ", version=" + + this.version + ", format='" + this.format + '\'' + '}'; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaRegistrationResponse.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaRegistrationResponse.java new file mode 100644 index 000000000..a168e40e1 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/SchemaRegistrationResponse.java @@ -0,0 +1,44 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry; + +/** + * @author Marius Bogoevici + */ +public class SchemaRegistrationResponse { + + private int id; + + private SchemaReference schemaReference; + + public int getId() { + return this.id; + } + + public void setId(int id) { + this.id = id; + } + + public SchemaReference getSchemaReference() { + return this.schemaReference; + } + + public void setSchemaReference(SchemaReference schemaReference) { + this.schemaReference = schemaReference; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AbstractAvroMessageConverter.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AbstractAvroMessageConverter.java new file mode 100644 index 000000000..95ca6b846 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AbstractAvroMessageConverter.java @@ -0,0 +1,142 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; + +import org.springframework.core.io.Resource; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.AbstractMessageConverter; +import org.springframework.messaging.converter.MessageConversionException; +import org.springframework.util.MimeType; + +/** + * Base class for Apache Avro + * {@link org.springframework.messaging.converter.MessageConverter} implementations. + * + * @author Marius Bogoevici + * @author Vinicius Carvalho + * @author Sercan Karaoglu + * @author Ish Mahajan + */ +public abstract class AbstractAvroMessageConverter extends AbstractMessageConverter { + + /** + * common parser will let user to import external schemas. + */ + private Schema.Parser schemaParser = new Schema.Parser(); + private AvroSchemaServiceManager avroSchemaServiceManager; + + @Deprecated + protected AbstractAvroMessageConverter(MimeType supportedMimeType) { + this(Collections.singletonList(supportedMimeType), new AvroSchemaServiceManagerImpl()); + } + + protected AbstractAvroMessageConverter(MimeType supportedMimeType, AvroSchemaServiceManager avroSchemaServiceManager) { + this(Collections.singletonList(supportedMimeType), avroSchemaServiceManager); + } + + @Deprecated + protected AbstractAvroMessageConverter(Collection supportedMimeTypes) { + this(supportedMimeTypes, new AvroSchemaServiceManagerImpl()); + setContentTypeResolver(new OriginalContentTypeResolver()); + } + + protected AbstractAvroMessageConverter(Collection supportedMimeTypes, AvroSchemaServiceManager manager) { + super(supportedMimeTypes); + setContentTypeResolver(new OriginalContentTypeResolver()); + this.avroSchemaServiceManager = manager; + } + + protected AvroSchemaServiceManager avroSchemaServiceManager() { + return this.avroSchemaServiceManager; + } + + protected Schema parseSchema(Resource r) throws IOException { + return this.schemaParser.parse(r.getInputStream()); + } + + @Override + protected boolean canConvertFrom(Message message, Class targetClass) { + return super.canConvertFrom(message, targetClass) + && (message.getPayload() instanceof byte[]); + } + + @Override + protected Object convertFromInternal(Message message, Class targetClass, Object conversionHint) { + Object result; + try { + byte[] payload = (byte[]) message.getPayload(); + + MimeType mimeType = getContentTypeResolver().resolve(message.getHeaders()); + if (mimeType == null) { + if (conversionHint instanceof MimeType) { + mimeType = (MimeType) conversionHint; + } + else { + return null; + } + } + + Schema writerSchema = resolveWriterSchemaForDeserialization(mimeType); + Schema readerSchema = resolveReaderSchemaForDeserialization(targetClass); + + result = avroSchemaServiceManager().readData(targetClass, payload, readerSchema, writerSchema); + } + catch (IOException e) { + throw new MessageConversionException(message, "Failed to read payload", e); + } + return result; + } + + @Override + protected Object convertToInternal(Object payload, MessageHeaders headers, Object conversionHint) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + MimeType hintedContentType = null; + if (conversionHint instanceof MimeType) { + hintedContentType = (MimeType) conversionHint; + } + Schema schema = resolveSchemaForWriting(payload, headers, hintedContentType); + @SuppressWarnings("unchecked") + DatumWriter writer = avroSchemaServiceManager().getDatumWriter(payload.getClass(), schema); + Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); + writer.write(payload, encoder); + encoder.flush(); + } + catch (IOException e) { + throw new MessageConversionException("Failed to write payload", e); + } + return baos.toByteArray(); + } + + protected abstract Schema resolveSchemaForWriting(Object payload, MessageHeaders headers, MimeType hintedContentType); + + protected abstract Schema resolveWriterSchemaForDeserialization(MimeType mimeType); + + protected abstract Schema resolveReaderSchemaForDeserialization(Class targetClass); + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroMessageConverterAutoConfiguration.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroMessageConverterAutoConfiguration.java new file mode 100644 index 000000000..154ac60cd --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroMessageConverterAutoConfiguration.java @@ -0,0 +1,95 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import java.lang.reflect.Constructor; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.CacheManager; +import org.springframework.cache.concurrent.ConcurrentMapCacheManager; +import org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.util.ObjectUtils; +import org.springframework.util.ReflectionUtils; + +/** + * @author Marius Bogoevici + * @author Vinicius Carvalho + * @author Sercan Karaoglu + * @author Ish Mahajan + * @author Christian Tzolov + */ +@Configuration +@ConditionalOnClass(name = "org.apache.avro.Schema") +@ConditionalOnProperty(value = "spring.cloud.schemaRegistryClient.enabled", matchIfMissing = true) +@ConditionalOnBean(type = "org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient") +@EnableConfigurationProperties({ AvroMessageConverterProperties.class }) +@Import(AvroSchemaServiceManagerImpl.class) +public class AvroMessageConverterAutoConfiguration { + + @Bean + @ConditionalOnMissingBean(AvroSchemaRegistryClientMessageConverter.class) + public AvroSchemaRegistryClientMessageConverter avroSchemaMessageConverter1( + SchemaRegistryClient schemaRegistryClient, + AvroSchemaServiceManager avroSchemaServiceManager, + AvroMessageConverterProperties avroMessageConverterProperties) { + + AvroSchemaRegistryClientMessageConverter avroSchemaRegistryClientMessageConverter = + new AvroSchemaRegistryClientMessageConverter(schemaRegistryClient, cacheManager(), avroSchemaServiceManager); + + avroSchemaRegistryClientMessageConverter.setDynamicSchemaGenerationEnabled( + avroMessageConverterProperties.isDynamicSchemaGenerationEnabled()); + + if (avroMessageConverterProperties.getReaderSchema() != null) { + avroSchemaRegistryClientMessageConverter.setReaderSchema(avroMessageConverterProperties.getReaderSchema()); + } + if (!ObjectUtils.isEmpty(avroMessageConverterProperties.getSchemaLocations())) { + avroSchemaRegistryClientMessageConverter.setSchemaLocations(avroMessageConverterProperties.getSchemaLocations()); + } + if (!ObjectUtils.isEmpty(avroMessageConverterProperties.getSchemaImports())) { + avroSchemaRegistryClientMessageConverter.setSchemaImports(avroMessageConverterProperties.getSchemaImports()); + } + avroSchemaRegistryClientMessageConverter.setPrefix(avroMessageConverterProperties.getPrefix()); + + try { + Class clazz = avroMessageConverterProperties.getSubjectNamingStrategy(); + Constructor constructor = ReflectionUtils.accessibleConstructor(clazz); + avroSchemaRegistryClientMessageConverter.setSubjectNamingStrategy( + (SubjectNamingStrategy) constructor.newInstance()); + } + catch (Exception ex) { + throw new IllegalStateException("Unable to create SubjectNamingStrategy " + + avroMessageConverterProperties.getSubjectNamingStrategy().toString(), ex); + } + avroSchemaRegistryClientMessageConverter.setSubjectNamePrefix(avroMessageConverterProperties.getSubjectNamePrefix()); + + return avroSchemaRegistryClientMessageConverter; + } + + @Bean + @ConditionalOnMissingBean + public CacheManager cacheManager() { + return new ConcurrentMapCacheManager(); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroMessageConverterProperties.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroMessageConverterProperties.java new file mode 100644 index 000000000..237559482 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroMessageConverterProperties.java @@ -0,0 +1,115 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; + +/** + * @author Vinicius Carvalho + * @author Sercan Karaoglu + * @author Christian Tzolov + */ +@ConfigurationProperties(prefix = "spring.cloud.schema.avro") +public class AvroMessageConverterProperties { + + private boolean dynamicSchemaGenerationEnabled; + + private Resource readerSchema; + + /** + * The source directory of Apache Avro schema. This schema is used by this converter. + * If this schema depends on other schemas consider defining those those dependent + * ones in the {@link #schemaImports} + * @parameter + */ + private Resource[] schemaLocations; + + /** + * A list of files or directories that should be loaded first thus making them + * importable by subsequent schemas. Note that imported files should not reference + * each other. + * @parameter + */ + private Resource[] schemaImports; + + private String prefix = "vnd"; + + private String subjectNamePrefix; + + private Class subjectNamingStrategy = DefaultSubjectNamingStrategy.class; + + public Resource getReaderSchema() { + return this.readerSchema; + } + + public void setReaderSchema(Resource readerSchema) { + Assert.notNull(readerSchema, "cannot be null"); + this.readerSchema = readerSchema; + } + + public Resource[] getSchemaLocations() { + return this.schemaLocations; + } + + public void setSchemaLocations(Resource[] schemaLocations) { + Assert.notEmpty(schemaLocations, "cannot be null"); + this.schemaLocations = schemaLocations; + } + + public boolean isDynamicSchemaGenerationEnabled() { + return this.dynamicSchemaGenerationEnabled; + } + + public void setDynamicSchemaGenerationEnabled(boolean dynamicSchemaGenerationEnabled) { + this.dynamicSchemaGenerationEnabled = dynamicSchemaGenerationEnabled; + } + + public String getPrefix() { + return this.prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + public Class getSubjectNamingStrategy() { + return this.subjectNamingStrategy; + } + + public void setSubjectNamingStrategy(Class subjectNamingStrategy) { + Assert.notNull(subjectNamingStrategy, "cannot be null"); + this.subjectNamingStrategy = subjectNamingStrategy; + } + + public Resource[] getSchemaImports() { + return this.schemaImports; + } + + public void setSchemaImports(Resource[] schemaImports) { + this.schemaImports = schemaImports; + } + + public String getSubjectNamePrefix() { + return subjectNamePrefix; + } + + public void setSubjectNamePrefix(String subjectNamePrefix) { + this.subjectNamePrefix = subjectNamePrefix; + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaMessageConverter.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaMessageConverter.java new file mode 100644 index 000000000..69247a08a --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaMessageConverter.java @@ -0,0 +1,152 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.avro.Schema; + +import org.springframework.core.io.Resource; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.Assert; +import org.springframework.util.MimeType; + +/** + * A {@link org.springframework.messaging.converter.MessageConverter} using Apache Avro. + * The schema for serializing and deserializing will be automatically inferred from the + * class for {@link org.apache.avro.specific.SpecificRecord} and regular classes, unless a + * specific schema is set, case in which that schema will be used instead. For converting + * to {@link org.apache.avro.generic.GenericRecord} targets, a schema must be set.s + * + * @author Marius Bogoevici + * @author Ish Mahajan + */ + +public class AvroSchemaMessageConverter extends AbstractAvroMessageConverter { + + private Schema schema; + + /** + * Create a {@link AvroSchemaMessageConverter}. Uses the default {@link MimeType} of + * {@code "application/avro"}. + */ + @Deprecated + public AvroSchemaMessageConverter() { + super(new MimeType("application", "avro")); + } + + /** + * Create a {@link AvroSchemaMessageConverter}. Uses the default {@link MimeType} of + * {@code "application/avro"}. + * @param manager for schema management + */ + public AvroSchemaMessageConverter(AvroSchemaServiceManager manager) { + super(new MimeType("application", "avro"), manager); + } + + /** + * Create a {@link AvroSchemaMessageConverter}. The converter will be used for the + * provided {@link MimeType}. + * @param supportedMimeType mime type to be supported by + * {@link AvroSchemaMessageConverter} + */ + @Deprecated + public AvroSchemaMessageConverter(MimeType supportedMimeType) { + super(supportedMimeType); + } + + /** + * Create a {@link AvroSchemaMessageConverter}. The converter will be used for the + * provided {@link MimeType}. + * @param supportedMimeType mime type to be supported by + * {@link AvroSchemaMessageConverter} + * @param manager for schema management + */ + public AvroSchemaMessageConverter(MimeType supportedMimeType, AvroSchemaServiceManager manager) { + super(supportedMimeType, manager); + } + + /** + * Create a {@link AvroSchemaMessageConverter}. The converter will be used for the + * provided {@link MimeType}s. + * @param supportedMimeTypes the mime types supported by this converter + */ + @Deprecated + public AvroSchemaMessageConverter(Collection supportedMimeTypes) { + super(supportedMimeTypes); + } + + /** + * Create a {@link AvroSchemaMessageConverter}. The converter will be used for the + * provided {@link MimeType}s. + * @param supportedMimeTypes the mime types supported by this converter + * @param manager for schema management + */ + public AvroSchemaMessageConverter(Collection supportedMimeTypes, AvroSchemaServiceManager manager) { + super(supportedMimeTypes, manager); + } + + public Schema getSchema() { + return this.schema; + } + + /** + * Sets the Apache Avro schema to be used by this converter. + * @param schema schema to be used by this converter + */ + public void setSchema(Schema schema) { + Assert.notNull(schema, "schema cannot be null"); + this.schema = schema; + } + + /** + * The location of the Apache Avro schema to be used by this converter. + * @param schemaLocation the location of the schema used by this converter. + */ + public void setSchemaLocation(Resource schemaLocation) { + Assert.notNull(schemaLocation, "schema cannot be null"); + try { + this.schema = parseSchema(schemaLocation); + } + catch (IOException e) { + throw new IllegalStateException("Schema cannot be parsed:", e); + } + } + + @Override + protected boolean supports(Class clazz) { + return true; + } + + @Override + protected Schema resolveWriterSchemaForDeserialization(MimeType mimeType) { + return this.schema; + } + + @Override + protected Schema resolveReaderSchemaForDeserialization(Class targetClass) { + return this.schema; + } + + @Override + protected Schema resolveSchemaForWriting(Object payload, MessageHeaders headers, + MimeType hintedContentType) { + return this.schema; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaRegistryClientMessageConverter.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaRegistryClientMessageConverter.java new file mode 100644 index 000000000..18696db54 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaRegistryClientMessageConverter.java @@ -0,0 +1,413 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericContainer; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; +import org.springframework.cache.support.NoOpCacheManager; +import org.springframework.cloud.stream.schema.registry.ParsedSchema; +import org.springframework.cloud.stream.schema.registry.SchemaNotFoundException; +import org.springframework.cloud.stream.schema.registry.SchemaReference; +import org.springframework.cloud.stream.schema.registry.SchemaRegistrationResponse; +import org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient; +import org.springframework.core.io.Resource; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.Assert; +import org.springframework.util.MimeType; +import org.springframework.util.ObjectUtils; + +/** + * A {@link org.springframework.messaging.converter.MessageConverter} for Apache Avro, + * with the ability to publish and retrieve schemas stored in a schema server, allowing + * for schema evolution in applications. The supported content types are in the form + * `application/*+avro`. + * + * During the conversion to a message, the converter will set the 'contentType' header to + * 'application/[prefix].[subject].v[version]+avro', where: + * + *
  • + *
      + * prefix is a configurable prefix (default 'vnd'); + *
    + *
      + * subject is a subject derived from the type of the outgoing object - typically + * the class name; + *
    + *
      + * version is the schema version for the given subject; + *
    + *
    • + * + * When converting from a message, the converter will parse the content-type and use it to + * fetch and cache the writer schema using the provided {@link SchemaRegistryClient}. + * + * @author Marius Bogoevici + * @author Vinicius Carvalho + * @author Oleg Zhurakousky + * @author Sercan Karaoglu + * @author Ish Mahajan + */ +public class AvroSchemaRegistryClientMessageConverter extends AbstractAvroMessageConverter + implements InitializingBean { + + /** + * Avro format defined in the Mime type. + */ + public static final String AVRO_FORMAT = "avro"; + + /** + * Pattern for validating the prefix to be used in the publised subtype. + */ + public static final Pattern PREFIX_VALIDATION_PATTERN = Pattern.compile("[\\p{Alnum}]"); + + /** + * Spring Cloud Stream schema property prefix. + */ + public static final String CACHE_PREFIX = "org.springframework.cloud.stream.schema"; + + /** + * Property for reflection cache. + */ + public static final String REFLECTION_CACHE_NAME = CACHE_PREFIX + ".reflectionCache"; + + /** + * Property for schema cache. + */ + public static final String SCHEMA_CACHE_NAME = CACHE_PREFIX + ".schemaCache"; + + /** + * Property for reference cache. + */ + public static final String REFERENCE_CACHE_NAME = CACHE_PREFIX + ".referenceCache"; + + /** + * Default Mime type for Avro. + */ + public static final MimeType DEFAULT_AVRO_MIME_TYPE = new MimeType("application", "*+" + AVRO_FORMAT); + + private static final AvroSchemaServiceManager defaultAvroSchemaServiceManager = + new AvroSchemaServiceManagerImpl(); + + private final CacheManager cacheManager; + + protected Resource[] schemaImports = new Resource[] {}; + + private Pattern versionedSchema; + + private boolean dynamicSchemaGenerationEnabled; + + private Schema readerSchema; + + private Resource[] schemaLocations; + + private SchemaRegistryClient schemaRegistryClient; + + private String prefix = "vnd"; + + private String subjectNamePrefix; + + private SubjectNamingStrategy subjectNamingStrategy; + + /** + * Creates a new instance, configuring it with {@link SchemaRegistryClient} and + * {@link CacheManager}. + * @param schemaRegistryClient the {@link SchemaRegistryClient} used to interact with + * the schema registry server. + * @param cacheManager instance of {@link CacheManager} to cache parsed schemas. If + * caching is not required use {@link NoOpCacheManager} + */ + @Deprecated + public AvroSchemaRegistryClientMessageConverter( + SchemaRegistryClient schemaRegistryClient, CacheManager cacheManager) { + super(Collections.singletonList(DEFAULT_AVRO_MIME_TYPE), defaultAvroSchemaServiceManager); + Assert.notNull(schemaRegistryClient, "cannot be null"); + Assert.notNull(cacheManager, "'cacheManager' cannot be null"); + this.schemaRegistryClient = schemaRegistryClient; + this.cacheManager = cacheManager; + } + + /** + * Creates a new instance, configuring it with {@link SchemaRegistryClient} and + * {@link CacheManager}. + * @param schemaRegistryClient the {@link SchemaRegistryClient} used to interact with + * the schema registry server. + * @param cacheManager instance of {@link CacheManager} to cache parsed schemas. If + * caching is not required use {@link NoOpCacheManager} + * @param manager instance of {@link AvroSchemaServiceManager} to manage schemas. + */ + public AvroSchemaRegistryClientMessageConverter( + SchemaRegistryClient schemaRegistryClient, CacheManager cacheManager, AvroSchemaServiceManager manager) { + super(Collections.singletonList(DEFAULT_AVRO_MIME_TYPE), manager); + Assert.notNull(schemaRegistryClient, "cannot be null"); + Assert.notNull(cacheManager, "'cacheManager' cannot be null"); + Assert.notNull(manager, "'avroSchemaServiceManager' cannot be null"); + this.schemaRegistryClient = schemaRegistryClient; + this.cacheManager = cacheManager; + } + + public boolean isDynamicSchemaGenerationEnabled() { + return this.dynamicSchemaGenerationEnabled; + } + + /** + * Allows the converter to generate and register schemas automatically. If set to + * false, it only allows the converter to use pre-registered schemas. Default 'true'. + * @param dynamicSchemaGenerationEnabled true if dynamic schema generation is enabled + */ + public void setDynamicSchemaGenerationEnabled(boolean dynamicSchemaGenerationEnabled) { + this.dynamicSchemaGenerationEnabled = dynamicSchemaGenerationEnabled; + } + + /** + * A set of locations where the converter can load schemas from. Schemas provided at + * these locations will be registered automatically. + * @param schemaLocations array of locations + */ + public void setSchemaLocations(Resource[] schemaLocations) { + Assert.notEmpty(schemaLocations, "cannot be empty"); + this.schemaLocations = schemaLocations; + } + + /** + * A set of schema locations where should be imported first. Schemas provided at these + * locations will be reference, thus they should not reference each other. + * @param schemaImports array of schema imports + */ + public void setSchemaImports(Resource[] schemaImports) { + this.schemaImports = schemaImports; + } + + /** + * Set the prefix to be used in the published subtype. Default 'vnd'. + * @param prefix prefix to be set + */ + public void setPrefix(String prefix) { + Assert.hasText(prefix, "Prefix cannot be empty"); + Assert.isTrue(!PREFIX_VALIDATION_PATTERN.matcher(this.prefix).matches(), "Invalid prefix:" + this.prefix); + this.prefix = prefix; + } + + public void setReaderSchema(Resource readerSchema) { + Assert.notNull(readerSchema, "cannot be null"); + try { + this.readerSchema = parseSchema(readerSchema); + } + catch (IOException e) { + throw new BeanInitializationException("Cannot initialize reader schema", e); + } + } + + public void setSubjectNamingStrategy(SubjectNamingStrategy subjectNamingStrategy) { + this.subjectNamingStrategy = subjectNamingStrategy; + } + + public void setSubjectNamePrefix(String subjectNamePrefix) { + this.subjectNamePrefix = subjectNamePrefix; + } + + @Override + public void afterPropertiesSet() { + this.versionedSchema = Pattern.compile("application/" + this.prefix + + "\\.([\\p{Alnum}\\$\\.]+)\\.v(\\p{Digit}+)\\+" + AVRO_FORMAT); + + Stream.of(this.schemaImports, this.schemaLocations) + .filter(arr -> !ObjectUtils.isEmpty(arr)) + .distinct() + .peek(resources -> { + if (this.logger.isInfoEnabled()) { + this.logger.info("Scanning avro schema resources on classpath"); + this.logger.info("Parsing " + this.schemaImports.length + " schemas"); + } + }) + .flatMap(Arrays::stream) + .forEach(resource -> { + try { + Schema schema = parseSchema(resource); + if (schema.getType().equals(Schema.Type.UNION)) { + schema.getTypes().forEach(innerSchema -> registerSchema(resource, innerSchema)); + } + else { + registerSchema(resource, schema); + } + } + catch (IOException e) { + if (this.logger.isWarnEnabled()) { + this.logger.warn("Failed to parse schema at " + resource.getFilename(), e); + } + } + }); + + if (this.cacheManager instanceof NoOpCacheManager) { + this.logger.warn("Schema caching is effectively disabled " + + "since configured cache manager is a NoOpCacheManager. If this was not " + + "the intention, please provide the appropriate instance of CacheManager " + + "(i.e., ConcurrentMapCacheManager)."); + } + } + + protected String toSubject(String subjectNamePrefix, Schema schema) { + return this.subjectNamingStrategy.toSubject(subjectNamePrefix, schema); + } + + @Override + protected boolean supports(Class clazz) { + // we support all types + return true; + } + + @Override + protected boolean supportsMimeType(MessageHeaders headers) { + if (super.supportsMimeType(headers)) { + return true; + } + MimeType mimeType = getContentTypeResolver().resolve(headers); + return DEFAULT_AVRO_MIME_TYPE.includes(mimeType); + } + + @Override + protected Schema resolveSchemaForWriting(Object payload, MessageHeaders headers, + MimeType hintedContentType) { + + Schema schema; + schema = extractSchemaForWriting(payload); + ParsedSchema parsedSchema = this.getCache(REFERENCE_CACHE_NAME).get(schema, ParsedSchema.class); + + if (parsedSchema == null) { + parsedSchema = new ParsedSchema(schema); + this.getCache(REFERENCE_CACHE_NAME).putIfAbsent(schema, parsedSchema); + } + + if (parsedSchema.getRegistration() == null) { + SchemaRegistrationResponse response = this.schemaRegistryClient.register(toSubject(this.subjectNamePrefix, schema), + AVRO_FORMAT, parsedSchema.getRepresentation()); + parsedSchema.setRegistration(response); + + } + + SchemaReference schemaReference = parsedSchema.getRegistration().getSchemaReference(); + + DirectFieldAccessor dfa = new DirectFieldAccessor(headers); + @SuppressWarnings("unchecked") + Map _headers = (Map) dfa.getPropertyValue("headers"); + _headers.put(MessageHeaders.CONTENT_TYPE, "application/" + this.prefix + "." + schemaReference.getSubject() + + ".v" + schemaReference.getVersion() + "+" + AVRO_FORMAT); + + return schema; + } + + @Override + protected Schema resolveWriterSchemaForDeserialization(MimeType mimeType) { + SchemaReference schemaReference = extractSchemaReference(mimeType); + if (schemaReference != null) { + ParsedSchema parsedSchema = this.getCache(REFERENCE_CACHE_NAME).get(schemaReference, ParsedSchema.class); + if (parsedSchema == null) { + String schemaContent = this.schemaRegistryClient.fetch(schemaReference); + if (schemaContent != null) { + Schema schema = new Schema.Parser().parse(schemaContent); + parsedSchema = new ParsedSchema(schema); + this.getCache(REFERENCE_CACHE_NAME).putIfAbsent(schemaReference, parsedSchema); + } + } + if (parsedSchema != null) { + return parsedSchema.getSchema(); + } + } + return this.readerSchema; + } + + @Override + protected Schema resolveReaderSchemaForDeserialization(Class targetClass) { + return this.readerSchema; + } + + private Schema extractSchemaForWriting(Object payload) { + Schema schema = null; + if (this.logger.isDebugEnabled()) { + this.logger.debug("Obtaining schema for class " + payload.getClass()); + } + if (GenericContainer.class.isAssignableFrom(payload.getClass())) { + schema = ((GenericContainer) payload).getSchema(); + if (this.logger.isDebugEnabled()) { + this.logger.debug("Avro type detected, using schema from object"); + } + } + else { + schema = this.getCache(REFLECTION_CACHE_NAME).get(payload.getClass().getName(), Schema.class); + if (schema == null) { + if (!isDynamicSchemaGenerationEnabled()) { + throw new SchemaNotFoundException(String.format( + "No schema found in the local cache for %s, and dynamic schema generation is not enabled", + payload.getClass())); + } + else { + schema = super.avroSchemaServiceManager().getSchema(payload.getClass()); + } + this.getCache(REFLECTION_CACHE_NAME).put(payload.getClass().getName(), schema); + } + } + return schema; + } + + private void registerSchema(Resource schemaLocation, Schema schema) { + if (this.logger.isInfoEnabled()) { + this.logger.info("Resource " + schemaLocation.getFilename() + " parsed into schema " + + schema.getNamespace() + "." + schema.getName()); + } + + this.schemaRegistryClient.register(toSubject(this.subjectNamePrefix, schema), AVRO_FORMAT, schema.toString()); + + if (this.logger.isInfoEnabled()) { + this.logger.info("Schema " + schema.getName() + " registered with id " + schema); + } + + this.getCache(REFLECTION_CACHE_NAME).put(schema.getNamespace() + "." + schema.getName(), schema); + } + + private SchemaReference extractSchemaReference(MimeType mimeType) { + SchemaReference schemaReference = null; + Matcher schemaMatcher = this.versionedSchema.matcher(mimeType.toString()); + if (schemaMatcher.find()) { + String subject = schemaMatcher.group(1); + Integer version = Integer.parseInt(schemaMatcher.group(2)); + schemaReference = new SchemaReference(subject, version, AVRO_FORMAT); + } + return schemaReference; + } + + private Cache getCache(String name) { + Cache cache = this.cacheManager.getCache(name); + Assert.notNull(cache, "Cache by the name '" + name + "' is not present in this CacheManager - '" + + this.cacheManager + "'. Typically caches are auto-created by the CacheManagers. " + + "Consider reporting it as an issue to the developer of this CacheManager."); + return cache; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaServiceManager.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaServiceManager.java new file mode 100644 index 000000000..638857f39 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaServiceManager.java @@ -0,0 +1,75 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import java.io.IOException; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; + +/** + * Manage a {@link Schema} together with its String representation. + * + * Helps to substitute the default implementation of {@link org.apache.avro.Schema} + * Generation using Custom Avro schema generator + * + * Provide a custom bean definition of {@link AvroSchemaServiceManager} and mark + * it as @Primary to override the default implementation + * + * @author Ish Mahajan + * + */ +public interface AvroSchemaServiceManager { + + /** + * get {@link Schema}. + * @param clazz {@link Class} for which schema generation is required + * @return returns avro schema for given class + */ + Schema getSchema(Class clazz); + + /** + * get {@link DatumWriter}. + * @param type {@link Class} of java object which needs to be serialized + * @param schema {@link Schema} of object which needs to be serialized + * @return datum writer which can be used to write Avro payload + */ + DatumWriter getDatumWriter(Class type, Schema schema); + + /** + * get {@link DatumReader}. + * @param type {@link Class} of java object which needs to be serialized + * @param schema {@link Schema} default schema of object which needs to be de-serialized + * @param writerSchema {@link Schema} writerSchema provided at run time + * @return datum reader which can be used to read Avro payload + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + DatumReader getDatumReader(Class type, Schema schema, Schema writerSchema); + + /** + * read data from avro type payload {@link DatumReader}. + * @param targetClass {@link Class} of java object which needs to be serialized + * @param payload {@link byte} serialized payload of object which needs to be de-serialized + * @param readerSchema {@link Schema} readerSchema of object which needs to be de-serialized + * @param writerSchema {@link Schema} writerSchema used to while serializing payload + * @return java object after reading Avro Payload + * @throws IOException in case of error + */ + Object readData(Class targetClass, byte[] payload, Schema readerSchema, Schema writerSchema) + throws IOException; +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaServiceManagerImpl.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaServiceManagerImpl.java new file mode 100644 index 000000000..cb68765cb --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/AvroSchemaServiceManagerImpl.java @@ -0,0 +1,172 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import java.io.IOException; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.reflect.ReflectData; +import org.apache.avro.reflect.ReflectDatumReader; +import org.apache.avro.reflect.ReflectDatumWriter; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecord; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.messaging.converter.MessageConversionException; +import org.springframework.stereotype.Component; + +/** + * Default Concrete implementation of {@link AvroSchemaServiceManager}. + * + * Helps to substitute the default implementation of {@link org.apache.avro.Schema} Generation using Custom Avro + * schema generator + * + * Provide a custom bean definition of {@link AvroSchemaServiceManager} and mark it as @Primary to override this + * default implementation + * + * @author Ish Mahajan + * + */ + +@Component +public class AvroSchemaServiceManagerImpl implements AvroSchemaServiceManager { + + protected final Log logger = LogFactory.getLog(this.getClass()); + + /** + * get {@link Schema}. + * @param clazz {@link Class} for which schema generation is required + * @return returns avro schema for given class + */ + @Override + public Schema getSchema(Class clazz) { + return ReflectData.get().getSchema(clazz); + } + + /** + * get {@link DatumWriter}. + * @param type {@link Class} of java object which needs to be serialized + * @param schema {@link Schema} of object which needs to be serialized + * @return datum writer which can be used to write Avro payload + */ + @Override + public DatumWriter getDatumWriter(Class type, Schema schema) { + DatumWriter writer; + this.logger.debug("Finding correct DatumWriter for type " + type.getName()); + if (SpecificRecord.class.isAssignableFrom(type)) { + if (schema != null) { + writer = new SpecificDatumWriter<>(schema); + } + else { + writer = new SpecificDatumWriter(type); + } + } + else if (GenericRecord.class.isAssignableFrom(type)) { + writer = new GenericDatumWriter<>(schema); + } + else { + if (schema != null) { + writer = new ReflectDatumWriter<>(schema); + } + else { + writer = new ReflectDatumWriter(type); + } + } + return writer; + } + + /** + * get {@link DatumReader}. + * @param type {@link Class} of java object which needs to be serialized + * @param readerSchema {@link Schema} default schema of object which needs to be de-serialized + * @param writerSchema {@link Schema} writerSchema provided at run time + * @return datum reader which can be used to read Avro payload + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + @Override + public DatumReader getDatumReader(Class type, Schema readerSchema, Schema writerSchema) { + DatumReader reader = null; + if (SpecificRecord.class.isAssignableFrom(type)) { + if (readerSchema != null) { + if (writerSchema != null) { + reader = new SpecificDatumReader<>(writerSchema, readerSchema); + } + else { + reader = new SpecificDatumReader<>(readerSchema); + } + } + else { + reader = new SpecificDatumReader(type); + if (writerSchema != null) { + reader.setSchema(writerSchema); + } + } + } + else if (GenericRecord.class.isAssignableFrom(type)) { + if (readerSchema != null) { + if (writerSchema != null) { + reader = new GenericDatumReader<>(writerSchema, readerSchema); + } + else { + reader = new GenericDatumReader<>(readerSchema); + } + } + else { + if (writerSchema != null) { + reader = new GenericDatumReader(writerSchema); + } + } + } + else { + reader = new ReflectDatumReader(type); + if (writerSchema != null) { + reader.setSchema(writerSchema); + } + } + if (reader == null) { + throw new MessageConversionException("No schema can be inferred from type " + + type.getName() + " and no schema has been explicitly configured."); + } + return reader; + } + + /** + * read data from avro type payload {@link DatumReader}. + * @param clazz {@link Class} of java object which needs to be serialized + * @param payload {@link byte} serialized payload of object which needs to be de-serialized + * @param readerSchema {@link Schema} readerSchema of object which needs to be de-serialized + * @param writerSchema {@link Schema} writerSchema used to while serializing payload + * @return java object after reading Avro Payload + * @throws IOException is thrown in case of error + */ + @Override + public Object readData(Class clazz, byte[] payload, Schema readerSchema, Schema writerSchema) + throws IOException { + DatumReader reader = this.getDatumReader(clazz, readerSchema, writerSchema); + Decoder decoder = DecoderFactory.get().binaryDecoder(payload, null); + return reader.read(null, decoder); + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/DefaultSubjectNamingStrategy.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/DefaultSubjectNamingStrategy.java new file mode 100644 index 000000000..9377f6799 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/DefaultSubjectNamingStrategy.java @@ -0,0 +1,35 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import org.apache.avro.Schema; + +import org.springframework.util.StringUtils; + +/** + * @author David Kalosi + */ +public class DefaultSubjectNamingStrategy implements SubjectNamingStrategy { + + @Override + public String toSubject(String subjectNamePrefix, Schema schema) { + return StringUtils.hasText(subjectNamePrefix) ? + subjectNamePrefix + "-" + schema.getName().toLowerCase() : + schema.getName().toLowerCase(); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/OriginalContentTypeResolver.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/OriginalContentTypeResolver.java new file mode 100644 index 000000000..8274e12da --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/OriginalContentTypeResolver.java @@ -0,0 +1,60 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.ContentTypeResolver; +import org.springframework.util.MimeType; + +/** + * @author Vinicius Carvalho + * + * Resolves contentType looking for a originalContentType header first. If not found + * returns the contentType + * + */ +class OriginalContentTypeResolver implements ContentTypeResolver { + + private static final String BINDER_ORIGINAL_CONTENT_TYPE = "originalContentType"; + + private ConcurrentMap mimeTypeCache = new ConcurrentHashMap<>(); + + @Override + public MimeType resolve(MessageHeaders headers) { + Object contentType = headers + .get(BINDER_ORIGINAL_CONTENT_TYPE) != null + ? headers.get(BINDER_ORIGINAL_CONTENT_TYPE) + : headers.get(MessageHeaders.CONTENT_TYPE); + MimeType mimeType = null; + if (contentType instanceof MimeType) { + mimeType = (MimeType) contentType; + } + else if (contentType instanceof String) { + mimeType = this.mimeTypeCache.get(contentType); + if (mimeType == null) { + String valueAsString = (String) contentType; + mimeType = MimeType.valueOf(valueAsString); + this.mimeTypeCache.put(valueAsString, mimeType); + } + } + return mimeType; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/QualifiedSubjectNamingStrategy.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/QualifiedSubjectNamingStrategy.java new file mode 100644 index 000000000..01216d60a --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/QualifiedSubjectNamingStrategy.java @@ -0,0 +1,36 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import org.apache.avro.Schema; + +import org.springframework.util.StringUtils; + +/** + * @author José A. Íñigo + * @since 2.2.0 + */ +public class QualifiedSubjectNamingStrategy implements SubjectNamingStrategy { + + @Override + public String toSubject(String subjectNamePrefix, Schema schema) { + return StringUtils.hasText(subjectNamePrefix) ? + subjectNamePrefix + "-" + schema.getFullName().toLowerCase() : + schema.getFullName().toLowerCase(); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/SubjectNamingStrategy.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/SubjectNamingStrategy.java new file mode 100644 index 000000000..a0e07f2a4 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/SubjectNamingStrategy.java @@ -0,0 +1,37 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import org.apache.avro.Schema; + +/** + * Provides function towards naming schema registry subjects for Avro files. + * + * @author David Kalosi + */ +public interface SubjectNamingStrategy { + + /** + * Takes the Avro schema on input and returns the generated subject under which the + * schema should be registered. + * @param subjectNamePrefix optional subject name prefix + * @param schema schema to register + * @return subject name + */ + String toSubject(String subjectNamePrefix, Schema schema); + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/SubjectPrefixOnlyNamingStrategy.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/SubjectPrefixOnlyNamingStrategy.java new file mode 100644 index 000000000..ae63a7784 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/avro/SubjectPrefixOnlyNamingStrategy.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.avro; + +import org.apache.avro.Schema; + +/** + * @author Christian Tzolov + */ +public class SubjectPrefixOnlyNamingStrategy implements SubjectNamingStrategy { + + @Override + public String toSubject(String subjectNamePrefix, Schema schema) { + return subjectNamePrefix; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/CachingRegistryClient.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/CachingRegistryClient.java new file mode 100644 index 000000000..f6a144ad2 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/CachingRegistryClient.java @@ -0,0 +1,67 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.client; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.cloud.stream.schema.registry.SchemaReference; +import org.springframework.cloud.stream.schema.registry.SchemaRegistrationResponse; +import org.springframework.util.Assert; + +/** + * @author Vinicius Carvalho + */ +public class CachingRegistryClient implements SchemaRegistryClient { + + private static final String CACHE_PREFIX = "org.springframework.cloud.schema.registry.client"; + + private static final String ID_CACHE = CACHE_PREFIX + ".schemaByIdCache"; + + private static final String REF_CACHE = CACHE_PREFIX + ".schemaByReferenceCache"; + + private SchemaRegistryClient delegate; + + @Autowired + private CacheManager cacheManager; + + public CachingRegistryClient(SchemaRegistryClient delegate) { + Assert.notNull(delegate, "The delegate cannot be null"); + this.delegate = delegate; + } + + @Override + public SchemaRegistrationResponse register(String subject, String format, String schema) { + SchemaRegistrationResponse response = this.delegate.register(subject, format, schema); + this.cacheManager.getCache(ID_CACHE).put(response.getId(), schema); + this.cacheManager.getCache(REF_CACHE).put(response.getSchemaReference(), schema); + return response; + } + + @Override + @Cacheable(cacheNames = REF_CACHE) + public String fetch(SchemaReference schemaReference) { + return this.delegate.fetch(schemaReference); + } + + @Override + @Cacheable(cacheNames = ID_CACHE) + public String fetch(int id) { + return this.delegate.fetch(id); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/ConfluentSchemaRegistryClient.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/ConfluentSchemaRegistryClient.java new file mode 100644 index 000000000..f33cd5d5f --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/ConfluentSchemaRegistryClient.java @@ -0,0 +1,166 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.client; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.springframework.cloud.stream.schema.registry.SchemaNotFoundException; +import org.springframework.cloud.stream.schema.registry.SchemaReference; +import org.springframework.cloud.stream.schema.registry.SchemaRegistrationResponse; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestTemplate; + +/** + * @author Vinicius Carvalho + * @author Marius Bogoevici + * @author Jon Archer + * @author Tengzhou Dong + */ +public class ConfluentSchemaRegistryClient implements SchemaRegistryClient { + + private static final List ACCEPT_HEADERS = Arrays.asList( + "application/vnd.schemaregistry.v1+json", + "application/vnd.schemaregistry+json", "application/json"); + + private RestTemplate template; + + private String endpoint = "http://localhost:8081"; + + private ObjectMapper mapper; + + public ConfluentSchemaRegistryClient() { + this(new RestTemplate()); + } + + public ConfluentSchemaRegistryClient(RestTemplate template) { + this(template, new ObjectMapper()); + } + + public ConfluentSchemaRegistryClient(RestTemplate template, ObjectMapper mapper) { + this.template = template; + this.mapper = mapper; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + @Override + public SchemaRegistrationResponse register(String subject, String format, String schema) { + Assert.isTrue("avro".equals(format), "Only Avro is supported"); + HttpHeaders headers = new HttpHeaders(); + headers.put("Accept", ACCEPT_HEADERS); + headers.add("Content-Type", "application/json"); + Integer version = null; + Integer id = null; + String payload = null; + Map maps = new HashMap<>(); + maps.put("schema", schema); + try { + payload = this.mapper.writeValueAsString(maps); + } + catch (JsonProcessingException e) { + throw new RuntimeException("Could not parse schema, invalid JSON format", e); + } + try { + HttpEntity request = new HttpEntity<>(payload, headers); + ResponseEntity response = this.template.exchange( + this.endpoint + "/subjects/" + subject + "/versions", HttpMethod.POST, request, Map.class); + id = (Integer) response.getBody().get("id"); + } + catch (HttpStatusCodeException httpException) { + throw new RuntimeException(String.format("Failed to register subject %s, server replied with status %d", + subject, httpException.getStatusCode().value()), httpException); + } + + try { + ResponseEntity response = this.template.getForEntity( + this.endpoint + "/subjects/" + subject + "/versions", List.class); + + final List body = response.getBody(); + if (!CollectionUtils.isEmpty(body)) { + version = (Integer) body.get(body.size() - 1); + } + } + catch (HttpStatusCodeException httpException) { + throw new RuntimeException(String.format("Failed to register subject %s, server replied with status %d", + subject, httpException.getStatusCode().value()), httpException); + } + + SchemaRegistrationResponse schemaRegistrationResponse = new SchemaRegistrationResponse(); + schemaRegistrationResponse.setId(id); + schemaRegistrationResponse.setSchemaReference(new SchemaReference(subject, version, "avro")); + return schemaRegistrationResponse; + } + + @Override + public String fetch(SchemaReference schemaReference) { + String path = String.format("/subjects/%s/versions/%d", schemaReference.getSubject(), schemaReference.getVersion()); + HttpHeaders headers = new HttpHeaders(); + headers.put("Accept", ACCEPT_HEADERS); + headers.add("Content-Type", "application/vnd.schemaregistry.v1+json"); + HttpEntity request = new HttpEntity<>("", headers); + try { + ResponseEntity response = this.template.exchange(this.endpoint + path, + HttpMethod.GET, request, Map.class); + return (String) response.getBody().get("schema"); + } + catch (HttpStatusCodeException e) { + if (e.getStatusCode() == HttpStatus.NOT_FOUND) { + throw new SchemaNotFoundException(String.format("Could not find schema for reference: %s", schemaReference)); + } + else { + throw e; + } + } + } + + @Override + public String fetch(int id) { + String path = String.format("/schemas/ids/%d", id); + HttpHeaders headers = new HttpHeaders(); + headers.put("Accept", ACCEPT_HEADERS); + headers.add("Content-Type", "application/vnd.schemaregistry.v1+json"); + HttpEntity request = new HttpEntity<>("", headers); + try { + ResponseEntity response = this.template.exchange(this.endpoint + path, + HttpMethod.GET, request, Map.class); + return (String) response.getBody().get("schema"); + } + catch (HttpStatusCodeException e) { + if (e.getStatusCode() == HttpStatus.NOT_FOUND) { + throw new SchemaNotFoundException(String.format("Could not find schema with id: %s", id)); + } + else { + throw e; + } + } + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/DefaultSchemaRegistryClient.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/DefaultSchemaRegistryClient.java new file mode 100644 index 000000000..639a3d805 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/DefaultSchemaRegistryClient.java @@ -0,0 +1,104 @@ +/* + * Copyright 2016-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.client; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.cloud.stream.schema.registry.SchemaReference; +import org.springframework.cloud.stream.schema.registry.SchemaRegistrationResponse; +import org.springframework.http.ResponseEntity; +import org.springframework.util.Assert; +import org.springframework.web.client.RestTemplate; + +/** + * @author Marius Bogoevici + * @author Vinicius Carvalho + * @author Christian Tzolov + */ +public class DefaultSchemaRegistryClient implements SchemaRegistryClient { + + private RestTemplate restTemplate; + + private String endpoint = "http://localhost:8990"; + + public DefaultSchemaRegistryClient(RestTemplateBuilder builder) { + this(builder.build()); + } + + public DefaultSchemaRegistryClient(RestTemplate restTemplate) { + Assert.notNull(restTemplate, "'restTemplate' must not be null."); + this.restTemplate = restTemplate; + } + + protected String getEndpoint() { + return this.endpoint; + } + + public void setEndpoint(String endpoint) { + Assert.hasText(endpoint, "cannot be empty"); + this.endpoint = endpoint; + } + + protected RestTemplate getRestTemplate() { + return this.restTemplate; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Override + public SchemaRegistrationResponse register(String subject, String format, String schema) { + Map requestBody = new HashMap<>(); + requestBody.put("subject", subject); + requestBody.put("format", format); + requestBody.put("definition", schema); + ResponseEntity responseEntity = this.restTemplate.postForEntity(this.endpoint, requestBody, Map.class); + if (responseEntity.getStatusCode().is2xxSuccessful()) { + SchemaRegistrationResponse registrationResponse = new SchemaRegistrationResponse(); + Map responseBody = (Map) responseEntity.getBody(); + registrationResponse.setId((Integer) responseBody.get("id")); + registrationResponse.setSchemaReference(new SchemaReference(subject, (Integer) responseBody.get("version"), + responseBody.get("format").toString())); + return registrationResponse; + } + throw new RuntimeException( + "Failed to register schema: " + responseEntity.toString()); + } + + @SuppressWarnings("rawtypes") + @Override + public String fetch(SchemaReference schemaReference) { + ResponseEntity responseEntity = this.restTemplate.getForEntity(this.endpoint + + "/" + schemaReference.getSubject() + "/" + schemaReference.getFormat() + + "/v" + schemaReference.getVersion(), Map.class); + if (!responseEntity.getStatusCode().is2xxSuccessful()) { + throw new RuntimeException("Failed to fetch schema: " + responseEntity.toString()); + } + return (String) responseEntity.getBody().get("definition"); + } + + @SuppressWarnings("rawtypes") + @Override + public String fetch(int id) { + ResponseEntity responseEntity = this.restTemplate.getForEntity(this.endpoint + "/schemas/" + id, Map.class); + if (!responseEntity.getStatusCode().is2xxSuccessful()) { + throw new RuntimeException("Failed to fetch schema: " + responseEntity.toString()); + } + return (String) responseEntity.getBody().get("definition"); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/EnableSchemaRegistryClient.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/EnableSchemaRegistryClient.java new file mode 100644 index 000000000..206bed20a --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/EnableSchemaRegistryClient.java @@ -0,0 +1,41 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.client; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.cloud.stream.schema.registry.client.config.SchemaRegistryClientConfiguration; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * @author Marius Bogoevici + */ +@Target({ ElementType.TYPE, ElementType.ANNOTATION_TYPE }) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@Configuration +@Import(SchemaRegistryClientConfiguration.class) +public @interface EnableSchemaRegistryClient { + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/SchemaRegistryClient.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/SchemaRegistryClient.java new file mode 100644 index 000000000..1dd54020f --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/SchemaRegistryClient.java @@ -0,0 +1,54 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.client; + +import org.springframework.cloud.stream.schema.registry.SchemaReference; +import org.springframework.cloud.stream.schema.registry.SchemaRegistrationResponse; + +/** + * @author Vinicius Carvalho + * @author Marius Bogoevici + */ +public interface SchemaRegistryClient { + + /** + * Registers a schema with the remote repository returning the unique identifier + * associated with this schema. + * @param subject the full name of the schema + * @param format format of the schema + * @param schema string representation of the schema + * @return a {@link SchemaRegistrationResponse} representing the result of the + * operation + */ + SchemaRegistrationResponse register(String subject, String format, String schema); + + /** + * Retrieves a schema by its reference (subject and version). + * @param schemaReference a {@link SchemaReference} used to identify the target + * schema. + * @return schema + */ + String fetch(SchemaReference schemaReference); + + /** + * Retrieves a schema by its identifier. + * @param id the id of the target schema. + * @return schema + */ + String fetch(int id); + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/config/SchemaRegistryClientConfiguration.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/config/SchemaRegistryClientConfiguration.java new file mode 100644 index 000000000..5173255a0 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/config/SchemaRegistryClientConfiguration.java @@ -0,0 +1,55 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.client.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.cloud.stream.schema.registry.client.CachingRegistryClient; +import org.springframework.cloud.stream.schema.registry.client.DefaultSchemaRegistryClient; +import org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.util.StringUtils; + +/** + * @author Marius Bogoevici + * @author Vinicius Carvalho + * @author Soby Chacko + */ +@Configuration +@EnableConfigurationProperties(SchemaRegistryClientProperties.class) +public class SchemaRegistryClientConfiguration { + + @Bean + @ConditionalOnMissingBean + public SchemaRegistryClient schemaRegistryClient(SchemaRegistryClientProperties schemaRegistryClientProperties, + RestTemplateBuilder restTemplateBuilder) { + DefaultSchemaRegistryClient defaultSchemaRegistryClient = new DefaultSchemaRegistryClient(restTemplateBuilder); + + if (StringUtils.hasText(schemaRegistryClientProperties.getEndpoint())) { + defaultSchemaRegistryClient.setEndpoint(schemaRegistryClientProperties.getEndpoint()); + } + + SchemaRegistryClient client = (schemaRegistryClientProperties.isCached()) + ? new CachingRegistryClient(defaultSchemaRegistryClient) + : defaultSchemaRegistryClient; + + return client; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/config/SchemaRegistryClientProperties.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/config/SchemaRegistryClientProperties.java new file mode 100644 index 000000000..9a13896a8 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/java/org/springframework/cloud/stream/schema/registry/client/config/SchemaRegistryClientProperties.java @@ -0,0 +1,49 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.client.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * @author Marius Bogoevici + * @author Vinicius Carvalho + * @author Christian Tzolov + */ +@ConfigurationProperties(prefix = "spring.cloud.stream.schema-registry-client") +public class SchemaRegistryClientProperties { + + private String endpoint; + + private boolean cached = false; + + public String getEndpoint() { + return this.endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public boolean isCached() { + return this.cached; + } + + public void setCached(boolean cached) { + this.cached = cached; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/main/resources/META-INF/spring.factories b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/resources/META-INF/spring.factories new file mode 100644 index 000000000..6dd0ef937 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/main/resources/META-INF/spring.factories @@ -0,0 +1,3 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.springframework.cloud.stream.schema.registry.avro.AvroMessageConverterAutoConfiguration + diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaLocationsTest.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaLocationsTest.java new file mode 100644 index 000000000..f39d2ad3c --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaLocationsTest.java @@ -0,0 +1,122 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro; + +import java.io.IOException; +import java.util.UUID; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaMessageConverter; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManager; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManagerImpl; +import org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.Resource; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.MimeType; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +public class AvroSchemaLocationsTest { + + @Test + public void schemaLocationWithMultipleRecords() throws Exception { + testMultipleRecordsSchemaLoading( + "--spring.cloud.schema.avro.schema-locations=classpath:schemas/user1_multiple_records.schema"); + } + + @Test + public void schemaImportWithMultipleRecords() throws Exception { + testMultipleRecordsSchemaLoading( + "--spring.cloud.schema.avro.schema-imports=classpath:schemas/user1_multiple_records.schema"); + } + + private void testMultipleRecordsSchemaLoading(String schemaLoadingProperty) throws Exception { + User1 user1 = new User1(); + user1.setFavoriteColor("foo" + UUID.randomUUID()); + user1.setName("foo" + UUID.randomUUID()); + + org.springframework.cloud.stream.schema.avro.v2.User1 user2 = new org.springframework.cloud.stream.schema.avro.v2.User1(); + user2.setFavoriteColor("foo" + UUID.randomUUID().toString()); + user2.setName("foo" + UUID.randomUUID().toString()); + user2.setFavoritePlace("Amsterdam"); + + + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(AvroSourceApplication.class)) + .web(WebApplicationType.NONE).run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.myBinding-out-0.contentType=application/*+avro", schemaLoadingProperty)) { + StreamBridge streamBridge = context.getBean(StreamBridge.class); + + streamBridge.send("myBinding-out-0", MessageBuilder.withPayload(user1).build()); + streamBridge.send("myBinding-out-0", MessageBuilder.withPayload(user2).build()); + + OutputDestination output = context.getBean(OutputDestination.class); + Message result = output.receive(); + + final MessageConverter avroSchemaMessageConverter = (MessageConverter) context.getBean("userMessageConverter"); + final User1 receivedUser1 = (User1) avroSchemaMessageConverter.fromMessage(result, User1.class); + result = output.receive(); + final User1 receivedUser2 = (User1) avroSchemaMessageConverter.fromMessage(result, User1.class); + + assertThat(receivedUser1.getFavoriteColor()).isEqualTo(user1.getFavoriteColor()); + assertThat(receivedUser1.getName()).isEqualTo(user1.getName()); + + assertThat(receivedUser2.getFavoriteColor()).isEqualTo(user2.getFavoriteColor()); + assertThat(receivedUser2.getName()).isEqualTo(user2.getName()); + + } + } + + static SchemaRegistryClient stubSchemaRegistryClient = new StubSchemaRegistryClient(); + + @EnableAutoConfiguration + public static class AvroSourceApplication { + + private Resource schemaLocation; + + @Bean + public SchemaRegistryClient schemaRegistryClient() { + return stubSchemaRegistryClient; + } + + @Bean + public MessageConverter userMessageConverter() throws IOException { + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter( + MimeType.valueOf("application/avro"), manager); + if (this.schemaLocation != null) { + avroSchemaMessageConverter.setSchemaLocation(this.schemaLocation); + } + return avroSchemaMessageConverter; + } + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaMessageConverterTests.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaMessageConverterTests.java new file mode 100644 index 000000000..150525c23 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaMessageConverterTests.java @@ -0,0 +1,165 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro; + +import java.io.IOException; +import java.util.UUID; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaMessageConverter; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManager; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManagerImpl; +import org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.Resource; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.util.MimeType; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + + +/** + * @author Marius Bogoevici + */ +public class AvroSchemaMessageConverterTests { + + static StubSchemaRegistryClient stubSchemaRegistryClient = new StubSchemaRegistryClient(); + + @Test + public void testSendMessageWithLocation() throws Exception { + + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(AvroSourceApplication.class)) + .web(WebApplicationType.NONE).run("--server.port=0", + "--spring.jmx.enabled=false", + "--schemaLocation=classpath:schemas/users_v1.schema", + "--spring.cloud.stream.bindings.myBinding-out-0.contentType=avro/bytes")) { + StreamBridge streamBridge = context.getBean(StreamBridge.class); + + User1 user1 = new User1(); + user1.setName("foo" + UUID.randomUUID()); + user1.setFavoriteColor("foo" + UUID.randomUUID()); + + streamBridge.send("myBinding-out-0", user1); + + User2 user2 = new User2(); + user2.setFavoriteColor("foo" + UUID.randomUUID().toString()); + user2.setFavoritePlace("foo" + UUID.randomUUID().toString()); + user2.setName("foo" + UUID.randomUUID().toString()); + + streamBridge.send("myBinding-out-0", user2); + + OutputDestination output = context.getBean(OutputDestination.class); + Message result = output.receive(); + + final MessageConverter userMessageConverter = (MessageConverter) context.getBean("userMessageConverter"); + final User1 receivedUser1 = (User1) userMessageConverter.fromMessage(result, User1.class); + + assertThat(receivedUser1).isNotNull(); + assertThat(receivedUser1).isNotSameAs(user1); + assertThat(receivedUser1.getFavoriteColor()).isEqualTo(user1.getFavoriteColor()); + assertThat(receivedUser1.getName()).isEqualTo(user1.getName()); + + result = output.receive(); + + final User2 receivedUser2 = (User2) userMessageConverter.fromMessage(result, User2.class); + + assertThat(receivedUser2).isNotNull(); + assertThat(receivedUser2).isNotSameAs(user2); + assertThat(receivedUser2.getFavoriteColor()).isEqualTo(user2.getFavoriteColor()); + assertThat(receivedUser2.getFavoritePlace()).isEqualTo(user2.getFavoritePlace()); + assertThat(receivedUser2.getName()).isEqualTo(user2.getName()); + } + } + + @Test + public void testSendMessageWithoutLocation() throws Exception { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(AvroSourceApplication.class)) + .web(WebApplicationType.NONE).run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.myBinding-out-0.contentType=avro/bytes")) { + StreamBridge streamBridge = context.getBean(StreamBridge.class); + + User1 user1 = new User1(); + user1.setName("foo" + UUID.randomUUID()); + user1.setFavoriteColor("foo" + UUID.randomUUID()); + + streamBridge.send("myBinding-out-0", user1); + + User2 user2 = new User2(); + user2.setFavoriteColor("foo" + UUID.randomUUID().toString()); + user2.setFavoritePlace("foo" + UUID.randomUUID().toString()); + user2.setName("foo" + UUID.randomUUID().toString()); + + streamBridge.send("myBinding-out-0", user2); + + OutputDestination output = context.getBean(OutputDestination.class); + Message result = output.receive(); + + final MessageConverter userMessageConverter = (MessageConverter) context.getBean("userMessageConverter"); + final User1 receivedUser1 = (User1) userMessageConverter.fromMessage(result, User1.class); + + assertThat(receivedUser1).isNotNull(); + assertThat(receivedUser1).isNotSameAs(user1); + assertThat(receivedUser1.getFavoriteColor()).isEqualTo(user1.getFavoriteColor()); + assertThat(receivedUser1.getName()).isEqualTo(user1.getName()); + + result = output.receive(); + + final User2 receivedUser2 = (User2) userMessageConverter.fromMessage(result, User2.class); + + assertThat(receivedUser2).isNotNull(); + assertThat(receivedUser2).isNotSameAs(user2); + assertThat(receivedUser2.getFavoriteColor()).isEqualTo(user2.getFavoriteColor()); + assertThat(receivedUser2.getFavoritePlace()).isEqualTo(user2.getFavoritePlace()); + assertThat(receivedUser2.getName()).isEqualTo(user2.getName()); + } + } + + @EnableAutoConfiguration + public static class AvroSourceApplication { + + private Resource schemaLocation; + + @Bean + public SchemaRegistryClient schemaRegistryClient() { + return stubSchemaRegistryClient; + } + + @Bean + public MessageConverter userMessageConverter() throws IOException { + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter( + MimeType.valueOf("avro/bytes"), manager); + if (this.schemaLocation != null) { + avroSchemaMessageConverter.setSchemaLocation(this.schemaLocation); + } + return avroSchemaMessageConverter; + } + + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaServiceManagerTests.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaServiceManagerTests.java new file mode 100644 index 000000000..10614629d --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/AvroSchemaServiceManagerTests.java @@ -0,0 +1,190 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro; + + +import java.io.File; +import java.io.IOException; + +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.avro.AvroFactory; +import com.fasterxml.jackson.dataformat.avro.AvroMapper; +import com.fasterxml.jackson.dataformat.avro.AvroSchema; +import com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaGenerator; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.avro.file.DataFileReader; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.assertj.core.util.Lists; +import org.junit.jupiter.api.Test; + +import org.springframework.cloud.stream.schema.avro.domain.FoodOrder; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaMessageConverter; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManager; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManagerImpl; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.util.MimeType; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; + +/** + * @author Ish Mahajan + */ +public class AvroSchemaServiceManagerTests { + + private final Log logger = LogFactory.getLog(AvroSchemaServiceManagerTests.class); + + @Test + public void testWithDefaultImplementation() throws IOException { + assertThatThrownBy(() -> { + + AvroSchemaServiceManager defaultServiceManager = new AvroSchemaServiceManagerImpl(); + + Schema schema = defaultServiceManager.getSchema(FoodOrder.class); + FoodOrder foodOrder = new FoodOrder(); + foodOrder.setRestaurant("Spring Kitchen"); + foodOrder.setOrderDescription("avro makhani"); + foodOrder.setCustomerAddress("world wide web"); + File file = new File("foodorder.avro"); + + DatumWriter datumWriter = defaultServiceManager.getDatumWriter(foodOrder.getClass(), schema); + DataFileWriter dataFileWriter = new DataFileWriter(datumWriter); + dataFileWriter.create(schema, file); + dataFileWriter.append(foodOrder); + + FoodOrder foodOrder2 = new FoodOrder(); + dataFileWriter.append(foodOrder2); + dataFileWriter.close(); + + DatumReader userDatumReader = defaultServiceManager.getDatumReader(foodOrder.getClass(), schema, schema); + DataFileReader dataFileReader = new DataFileReader(file, userDatumReader); + FoodOrder foodOrderDeserialized = null; + while (dataFileReader.hasNext()) { + // Reuse user object by passing it to next(). This saves us from + // allocating and garbage collecting many objects for files with + // many items. + foodOrderDeserialized = dataFileReader.next(foodOrderDeserialized); + logger.info("De-serialised Successfully : " + foodOrderDeserialized); + } + }).isInstanceOf(DataFileWriter.AppendWriteException.class); + } + + @Test + public void testWithCustomImplementation() throws IOException { + AvroSchemaServiceManager manager = new AvroSchemaServiceManager() { + @Override + public Schema getSchema(Class clazz) { + ObjectMapper mapper = new ObjectMapper(new AvroFactory()); + AvroSchemaGenerator gen = new AvroSchemaGenerator(); + try { + mapper.acceptJsonFormatVisitor(FoodOrder.class, gen); + } + catch (JsonMappingException e) { + fail("Error while setting acceptJsonFormatVisitor {}", e); + } + AvroSchema schemaWrapper = gen.getGeneratedSchema(); + return schemaWrapper.getAvroSchema(); + } + + @Override + public DatumWriter getDatumWriter(Class type, Schema schema) { + return new AvroSchemaServiceManagerImpl().getDatumWriter(type, schema); + } + + @Override + public DatumReader getDatumReader(Class type, Schema schema, Schema writerSchema) { + return new AvroSchemaServiceManagerImpl().getDatumReader(type, schema, schema); + } + + @Override + public Object readData(Class targetClass, byte[] payload, Schema readerSchema, + Schema writerSchema) throws IOException { + ObjectMapper mapper = new ObjectMapper(new AvroFactory()); + AvroSchemaGenerator gen = new AvroSchemaGenerator(); + try { + mapper.acceptJsonFormatVisitor(targetClass, gen); + } + catch (JsonMappingException e) { + fail("Error while setting acceptJsonFormatVisitor {}", e); + } + return mapper.readerFor(targetClass) + .with(new AvroSchema(readerSchema)) + .readValue(payload); + } + }; + + FoodOrder foodOrder1 = new FoodOrder(); + foodOrder1.setRestaurant("Spring Kitchen"); + foodOrder1.setOrderDescription("avro makhani"); + foodOrder1.setCustomerAddress("world wide web"); + FoodOrder foodOrder2 = new FoodOrder(); + foodOrder2.setRestaurant("Spring Kitchen"); + + Schema schema = manager.getSchema(FoodOrder.class); + AvroMapper mapper = new AvroMapper(); + byte[] payload1 = mapper.writer(new AvroSchema(schema)).writeValueAsBytes(foodOrder1); + byte[] payload2 = mapper.writer(new AvroSchema(schema)).writeValueAsBytes(foodOrder2); + foodOrder1 = (FoodOrder) manager.readData(foodOrder1.getClass(), payload1, schema, schema); + foodOrder2 = (FoodOrder) manager.readData(foodOrder1.getClass(), payload2, schema, schema); + assertThat(foodOrder2.getOrderDescription()).isNull(); + assertThat(foodOrder2.getCustomerAddress()).isNull(); + } + + @Test + public void testAvroSchemaMessageConverter() { + AvroSchemaMessageConverter converter = new AvroSchemaMessageConverter(); + MimeType mimeType = new MimeType("application", "avro"); + assertThat(mimeType).isEqualTo(converter.getSupportedMimeTypes().get(0)); + + AvroSchemaMessageConverter converter2 = new AvroSchemaMessageConverter(mimeType); + assertThat(mimeType).isEqualTo(converter2.getSupportedMimeTypes().get(0)); + + AvroSchemaMessageConverter converter3 = + new AvroSchemaMessageConverter(Lists.newArrayList(mimeType)); + assertThat(mimeType).isEqualTo(converter3.getSupportedMimeTypes().get(0)); + + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaMessageConverter converter4 = new AvroSchemaMessageConverter(manager); + assertThat(mimeType).isEqualTo(converter4.getSupportedMimeTypes().get(0)); + + AvroSchemaMessageConverter converter5 = + new AvroSchemaMessageConverter(Lists.newArrayList(mimeType), manager); + Schema schema = manager.getSchema(FoodOrder.class); + converter5.setSchema(schema); + assertThat(mimeType).isEqualTo(converter5.getSupportedMimeTypes().get(0)); + assertThat(schema).isEqualTo(converter5.getSchema()); + } + + @Test + public void testAvroSchemaMessageConverterException() { + assertThatThrownBy(() -> { + MimeType mimeType = new MimeType("application", "avro"); + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaMessageConverter converter = + new AvroSchemaMessageConverter(Lists.newArrayList(mimeType), manager); + converter.setSchemaLocation(new ByteArrayResource(new byte[2]) { + }); + }).isInstanceOf(SchemaParseException.class); + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/StubSchemaRegistryClient.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/StubSchemaRegistryClient.java new file mode 100644 index 000000000..70c549d87 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/StubSchemaRegistryClient.java @@ -0,0 +1,114 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.springframework.cloud.stream.schema.registry.SchemaNotFoundException; +import org.springframework.cloud.stream.schema.registry.SchemaReference; +import org.springframework.cloud.stream.schema.registry.SchemaRegistrationResponse; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaRegistryClientMessageConverter; +import org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient; + +/** + * @author Marius Bogoevici + */ +public class StubSchemaRegistryClient implements SchemaRegistryClient { + + private final AtomicInteger index = new AtomicInteger(0); + + private final Map schemasById = new HashMap<>(); + + private final Map> storedSchemas = new HashMap<>(); + + @Override + public SchemaRegistrationResponse register(String subject, String format, + String schema) { + if (!this.storedSchemas.containsKey(subject)) { + this.storedSchemas.put(subject, new TreeMap()); + } + Map schemaVersions = this.storedSchemas.get(subject); + for (Map.Entry integerSchemaEntry : schemaVersions + .entrySet()) { + + if (integerSchemaEntry.getValue().getSchema().equals(schema)) { + SchemaRegistrationResponse schemaRegistrationResponse = new SchemaRegistrationResponse(); + schemaRegistrationResponse.setId(integerSchemaEntry.getValue().getId()); + schemaRegistrationResponse.setSchemaReference( + new SchemaReference(subject, integerSchemaEntry.getKey(), + AvroSchemaRegistryClientMessageConverter.AVRO_FORMAT)); + return schemaRegistrationResponse; + } + } + int nextVersion = schemaVersions.size() + 1; + int id = this.index.incrementAndGet(); + schemaVersions.put(nextVersion, new SchemaWithId(id, schema)); + SchemaRegistrationResponse schemaRegistrationResponse = new SchemaRegistrationResponse(); + schemaRegistrationResponse.setId(this.index.getAndIncrement()); + schemaRegistrationResponse.setSchemaReference(new SchemaReference(subject, + nextVersion, AvroSchemaRegistryClientMessageConverter.AVRO_FORMAT)); + this.schemasById.put(id, schema); + return schemaRegistrationResponse; + } + + @Override + public String fetch(SchemaReference schemaReference) { + if (!AvroSchemaRegistryClientMessageConverter.AVRO_FORMAT + .equals(schemaReference.getFormat())) { + throw new IllegalArgumentException("Only 'avro' is supported by this client"); + } + if (!this.storedSchemas.containsKey(schemaReference.getSubject())) { + throw new SchemaNotFoundException("Not found: " + schemaReference); + } + if (!this.storedSchemas.get(schemaReference.getSubject()) + .containsKey(schemaReference.getVersion())) { + throw new SchemaNotFoundException("Not found: " + schemaReference); + } + return this.storedSchemas.get(schemaReference.getSubject()) + .get(schemaReference.getVersion()).getSchema(); + } + + @Override + public String fetch(int id) { + return this.schemasById.get(id); + } + + static class SchemaWithId { + + int id; + + String schema; + + SchemaWithId(int id, String schema) { + this.id = id; + this.schema = schema; + } + + public int getId() { + return this.id; + } + + public String getSchema() { + return this.schema; + } + + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/SubjectNamingStrategyTest.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/SubjectNamingStrategyTest.java new file mode 100644 index 000000000..08c1ffea4 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/SubjectNamingStrategyTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro; + +import java.io.IOException; +import java.util.UUID; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaMessageConverter; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManager; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManagerImpl; +import org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.Resource; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.util.MimeType; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author David Kalosi + * @author José A. Íñigo + * @author Christian Tzolov + */ +public class SubjectNamingStrategyTest { + + private static StubSchemaRegistryClient stubSchemaRegistryClient = new StubSchemaRegistryClient(); + + @Test + public void testQualifiedSubjectNamingStrategy() throws Exception { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(AvroSourceApplication.class)) + .web(WebApplicationType.NONE).run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.myBinding-out-0.contentType=application/*+avro", + "--spring.cloud.stream.schema.avro.subjectNamingStrategy=" + + "org.springframework.cloud.schema.registry.avro.QualifiedSubjectNamingStrategy", + "--spring.cloud.stream.schema.avro.dynamicSchemaGenerationEnabled=true")) { + StreamBridge streamBridge = context.getBean(StreamBridge.class); + + User1 user1 = new User1(); + user1.setFavoriteColor("foo" + UUID.randomUUID()); + user1.setName("foo" + UUID.randomUUID()); + + streamBridge.send("myBinding-out-0", user1); + + OutputDestination output = context.getBean(OutputDestination.class); + Message result = output.receive(); + + final MessageConverter userMessageConverter = (MessageConverter) context.getBean("userMessageConverter"); + final User1 receivedUser1 = (User1) userMessageConverter.fromMessage(result, User1.class); + + assertThat(receivedUser1).isNotNull(); + assertThat(receivedUser1).isNotSameAs(user1); + assertThat(receivedUser1.getFavoriteColor()).isEqualTo(user1.getFavoriteColor()); + assertThat(receivedUser1.getName()).isEqualTo(user1.getName()); + +// assertThat(result.getHeaders().get("contentType")).isEqualTo(MimeType.valueOf( +// "application/vnd.org.springframework.cloud.schema.avro.User1.v1+avro")); + + } + } + + @EnableAutoConfiguration + public static class AvroSourceApplication { + + private Resource schemaLocation; + + @Bean + public SchemaRegistryClient schemaRegistryClient() { + return stubSchemaRegistryClient; + } + + @Bean + public MessageConverter userMessageConverter() throws IOException { + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter( + MimeType.valueOf("application/avro"), manager); + if (this.schemaLocation != null) { + avroSchemaMessageConverter.setSchemaLocation(this.schemaLocation); + } + return avroSchemaMessageConverter; + } + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/User1.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/User1.java new file mode 100644 index 000000000..7abece21f --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/User1.java @@ -0,0 +1,58 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro; + +import org.apache.avro.reflect.Nullable; + +/** + * @author Marius Bogoevici + */ +public class User1 { + + @Nullable + private String name; + + private int favoriteNumber; + + @Nullable + private String favoriteColor; + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public int getFavoriteNumber() { + return this.favoriteNumber; + } + + public void setFavoriteNumber(int favoriteNumber) { + this.favoriteNumber = favoriteNumber; + } + + public String getFavoriteColor() { + return this.favoriteColor; + } + + public void setFavoriteColor(String favoriteColor) { + this.favoriteColor = favoriteColor; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/User2.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/User2.java new file mode 100644 index 000000000..705914347 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/User2.java @@ -0,0 +1,70 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro; + +import org.apache.avro.reflect.AvroDefault; +import org.apache.avro.reflect.Nullable; + +/** + * @author Marius Bogoevici + */ +public class User2 { + + @Nullable + private String name; + + private int favoriteNumber; + + @Nullable + private String favoriteColor; + + @AvroDefault("\"NYC\"") + private String favoritePlace = "Boston"; + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public int getFavoriteNumber() { + return this.favoriteNumber; + } + + public void setFavoriteNumber(int favoriteNumber) { + this.favoriteNumber = favoriteNumber; + } + + public String getFavoriteColor() { + return this.favoriteColor; + } + + public void setFavoriteColor(String favoriteColor) { + this.favoriteColor = favoriteColor; + } + + public String getFavoritePlace() { + return this.favoritePlace; + } + + public void setFavoritePlace(String favoritePlace) { + this.favoritePlace = favoritePlace; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/client/ConfluentSchemaRegistryClientTests.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/client/ConfluentSchemaRegistryClientTests.java new file mode 100644 index 000000000..fb3c0a1c9 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/client/ConfluentSchemaRegistryClientTests.java @@ -0,0 +1,206 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro.client; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.cloud.stream.schema.registry.SchemaNotFoundException; +import org.springframework.cloud.stream.schema.registry.SchemaReference; +import org.springframework.cloud.stream.schema.registry.SchemaRegistrationResponse; +import org.springframework.cloud.stream.schema.registry.client.ConfluentSchemaRegistryClient; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestTemplate; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.header; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withBadRequest; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +/** + * @author Vinicius Carvalho + * @author TengZhou Dong + */ +public class ConfluentSchemaRegistryClientTests { + + private RestTemplate restTemplate; + + private MockRestServiceServer mockRestServiceServer; + + @BeforeEach + public void setup() { + this.restTemplate = new RestTemplate(); + this.mockRestServiceServer = MockRestServiceServer + .createServer(this.restTemplate); + } + + @Test + public void registerSchema() throws Exception { + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/subjects/user/versions")) + .andExpect(method(HttpMethod.POST)) + .andExpect(header("Content-Type", "application/json")) + .andExpect(header("Accept", "application/vnd.schemaregistry.v1+json")) + .andRespond(withSuccess("{\"id\":101,\"version\":1}", MediaType.APPLICATION_JSON)); + + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/subjects/user/versions")) + .andExpect(method(HttpMethod.GET)) + .andRespond((withSuccess("[1]", MediaType.APPLICATION_JSON))); + + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient( + this.restTemplate); + SchemaRegistrationResponse response = client.register("user", "avro", "{}"); + assertThat(response.getSchemaReference().getVersion()).isEqualTo(1); + assertThat(response.getId()).isEqualTo(101); + this.mockRestServiceServer.verify(); + } + + @Test + public void registerWithInvalidJson() { + assertThatThrownBy(() -> { + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/subjects/user/versions")) + .andExpect(method(HttpMethod.POST)) + .andExpect(header("Content-Type", "application/json")) + .andExpect(header("Accept", "application/vnd.schemaregistry.v1+json")) + .andRespond(withBadRequest()); + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient( + this.restTemplate); + SchemaRegistrationResponse response = client.register("user", "avro", "<>"); + }).isInstanceOf(RuntimeException.class); + } + + @Test + public void registerIncompatibleSchema() { + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/subjects/user/versions")) + .andExpect(method(HttpMethod.POST)) + .andExpect(header("Content-Type", "application/json")) + .andExpect(header("Accept", "application/vnd.schemaregistry.v1+json")) + .andRespond(withStatus(HttpStatus.CONFLICT)); + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient( + this.restTemplate); + Exception expected = null; + try { + SchemaRegistrationResponse response = client.register("user", "avro", "{}"); + } + catch (Exception e) { + expected = e; + } + assertThat(expected instanceof RuntimeException).isTrue(); + assertThat(expected.getCause() instanceof HttpStatusCodeException).isTrue(); + this.mockRestServiceServer.verify(); + } + + @Test + public void findByReference() { + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/subjects/user/versions/1")) + .andExpect(method(HttpMethod.GET)) + .andExpect( + header("Content-Type", "application/vnd.schemaregistry.v1+json")) + .andExpect(header("Accept", "application/vnd.schemaregistry.v1+json")) + .andRespond(withSuccess("{\"schema\":\"\"}", MediaType.APPLICATION_JSON)); + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient( + this.restTemplate); + SchemaReference reference = new SchemaReference("user", 1, "avro"); + String schema = client.fetch(reference); + assertThat(schema).isEqualTo(""); + this.mockRestServiceServer.verify(); + } + + @Test + public void schemaNotFound() { + assertThatThrownBy(() -> { + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/subjects/user/versions/1")) + .andExpect(method(HttpMethod.GET)) + .andExpect( + header("Content-Type", "application/vnd.schemaregistry.v1+json")) + .andExpect(header("Accept", "application/vnd.schemaregistry.v1+json")) + .andRespond(withStatus(HttpStatus.NOT_FOUND)); + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient( + this.restTemplate); + SchemaReference reference = new SchemaReference("user", 1, "avro"); + client.fetch(reference); + }).isInstanceOf(SchemaNotFoundException.class); +} + + @Test + public void responseErrorFetch() { + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/subjects/user/versions")) + .andExpect(method(HttpMethod.POST)) + .andExpect(header("Content-Type", "application/json")) + .andExpect(header("Accept", "application/vnd.schemaregistry.v1+json")) + .andRespond(withBadRequest()); + + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient( + this.restTemplate); + Exception expected = null; + try { + SchemaRegistrationResponse response = client.register("user", "avro", "{}"); + } + catch (Exception e) { + expected = e; + } + assertThat(expected != null).isTrue(); + assertThat(expected.getCause() instanceof HttpStatusCodeException).isTrue(); + this.mockRestServiceServer.verify(); + } + + @Test + public void fetchById() { + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/schemas/ids/1")) + .andExpect(method(HttpMethod.GET)) + .andExpect( + header("Content-Type", "application/vnd.schemaregistry.v1+json")) + .andExpect(header("Accept", "application/vnd.schemaregistry.v1+json")) + .andRespond(withSuccess("{\"schema\":\"\"}", MediaType.APPLICATION_JSON)); + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient( + this.restTemplate); + String schema = client.fetch(1); + assertThat(schema).isEqualTo(""); + this.mockRestServiceServer.verify(); + } + + @Test + public void fetchByIdSchemaNotFound() { + assertThatThrownBy(() -> { + this.mockRestServiceServer + .expect(requestTo("http://localhost:8081/schemas/ids/1")) + .andExpect(method(HttpMethod.GET)) + .andExpect( + header("Content-Type", "application/vnd.schemaregistry.v1+json")) + .andExpect(header("Accept", "application/vnd.schemaregistry.v1+json")) + .andRespond(withStatus(HttpStatus.NOT_FOUND)); + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient( + this.restTemplate); + client.fetch(1); + }).isInstanceOf(SchemaNotFoundException.class); + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/domain/FoodOrder.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/domain/FoodOrder.java new file mode 100644 index 000000000..5dbd1b25b --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/domain/FoodOrder.java @@ -0,0 +1,44 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro.domain; + +/** + * @author Ish Mahajan + */ +public class FoodOrder { + private String restaurant; + private String customerAddress; + private String orderDescription; + public String getRestaurant() { + return restaurant; + } + public void setRestaurant(String restaurant) { + this.restaurant = restaurant; + } + public String getCustomerAddress() { + return customerAddress; + } + public void setCustomerAddress(String customerAddress) { + this.customerAddress = customerAddress; + } + public String getOrderDescription() { + return orderDescription; + } + public void setOrderDescription(String orderDescription) { + this.orderDescription = orderDescription; + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/v2/User1.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/v2/User1.java new file mode 100644 index 000000000..00138f730 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/avro/v2/User1.java @@ -0,0 +1,70 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.avro.v2; + +import org.apache.avro.reflect.AvroDefault; +import org.apache.avro.reflect.Nullable; + +/** + * @author Marius Bogoevici + */ +public class User1 { + + @Nullable + private String name; + + private int favoriteNumber; + + @Nullable + private String favoriteColor; + + @AvroDefault("\"NYC\"") + private String favoritePlace = "Boston"; + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public int getFavoriteNumber() { + return this.favoriteNumber; + } + + public void setFavoriteNumber(int favoriteNumber) { + this.favoriteNumber = favoriteNumber; + } + + public String getFavoriteColor() { + return this.favoriteColor; + } + + public void setFavoriteColor(String favoriteColor) { + this.favoriteColor = favoriteColor; + } + + public String getFavoritePlace() { + return this.favoritePlace; + } + + public void setFavoritePlace(String favoritePlace) { + this.favoritePlace = favoritePlace; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/serialization/AvroMessageConverterSerializationTests.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/serialization/AvroMessageConverterSerializationTests.java new file mode 100644 index 000000000..91e33b484 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/serialization/AvroMessageConverterSerializationTests.java @@ -0,0 +1,234 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.serialization; + +import java.io.ByteArrayOutputStream; +import java.util.Collections; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import example.avro.Command; +import example.avro.Email; +import example.avro.PushNotification; +import example.avro.Sms; +import example.avro.User; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.cache.support.NoOpCacheManager; +import org.springframework.cloud.stream.schema.registry.EnableSchemaRegistryServer; +import org.springframework.cloud.stream.schema.registry.SchemaReference; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaRegistryClientMessageConverter; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManager; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManagerImpl; +import org.springframework.cloud.stream.schema.registry.avro.DefaultSubjectNamingStrategy; +import org.springframework.cloud.stream.schema.registry.client.DefaultSchemaRegistryClient; +import org.springframework.cloud.stream.schema.registry.client.SchemaRegistryClient; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.support.MutableMessageHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.MimeType; +import org.springframework.util.MimeTypeUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Vinicius Carvalho + * @author Sercan Karaoglu + */ +public class AvroMessageConverterSerializationTests { + + Pattern versionedSchema = Pattern.compile( + "application/" + "vnd" + "\\.([\\p{Alnum}\\$\\.]+)\\.v(\\p{Digit}+)\\+avro"); + + Log logger = LogFactory.getLog(getClass()); + + private ConfigurableApplicationContext schemaRegistryServerContext; + + private RestTemplateBuilder restTemplateBuilder; + + public static Command notification() { + Command messageToSend = getCommandToSend(); + messageToSend.setType("notification"); + PushNotification pushNotification = new PushNotification(); + pushNotification.setArn("google"); + pushNotification.setText("hello"); + messageToSend.setPayload(pushNotification); + return messageToSend; + } + + public static Command sms() { + Command messageToSend = getCommandToSend(); + messageToSend.setType("sms"); + Sms sms = new Sms(); + sms.setPhoneNumber("6141231212"); + sms.setText("hello"); + messageToSend.setPayload(sms); + return messageToSend; + } + + public static Command email() { + Command messageToSend = getCommandToSend(); + messageToSend.setType("email"); + Email email = new Email(); + email.setAddressTo("sercan"); + email.setText("hello"); + email.setTitle("hi"); + messageToSend.setPayload(email); + return messageToSend; + } + + public static Command getCommandToSend() { + Command messageToSend = new Command(); + messageToSend.setCorrelationId("abc"); + return messageToSend; + } + + @BeforeEach + public void setup() { + this.schemaRegistryServerContext = SpringApplication.run( + ServerApplication.class, "--spring.main.allow-bean-definition-overriding=true"); + restTemplateBuilder = this.schemaRegistryServerContext.getBean(RestTemplateBuilder.class); + } + + @AfterEach + public void tearDown() { + this.schemaRegistryServerContext.close(); + } + + @Test + public void testSchemaImport() throws Exception { + SchemaRegistryClient client = new DefaultSchemaRegistryClient(restTemplateBuilder); + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaRegistryClientMessageConverter converter = new AvroSchemaRegistryClientMessageConverter( + client, new NoOpCacheManager(), manager); + converter.setSubjectNamingStrategy(new DefaultSubjectNamingStrategy()); + converter.setDynamicSchemaGenerationEnabled(false); + converter.setSchemaLocations(this.schemaRegistryServerContext + .getResources("classpath:schemas/Command.avsc")); + converter.setSchemaImports(this.schemaRegistryServerContext + .getResources("classpath:schemas/imports/*.avsc")); + converter.afterPropertiesSet(); + Command notification = notification(); + Message specificMessage = converter.toMessage(notification, + new MutableMessageHeaders(Collections.emptyMap())); + Object o = converter.fromMessage(specificMessage, Command.class); + + assertThat(o).isEqualTo(notification) + .as("Serialization issue when use schema-imports"); + } + + @Test + public void sourceWriteSameVersion() throws Exception { + User specificRecord = new User(); + specificRecord.setName("joe"); + Schema v1 = new Schema.Parser().parse(AvroMessageConverterSerializationTests.class + .getClassLoader().getResourceAsStream("schemas/user.avsc")); + GenericRecord genericRecord = new GenericData.Record(v1); + genericRecord.put("name", "joe"); + SchemaRegistryClient client = new DefaultSchemaRegistryClient(restTemplateBuilder); + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaRegistryClientMessageConverter converter = new AvroSchemaRegistryClientMessageConverter( + client, new NoOpCacheManager(), manager); + + converter.setSubjectNamingStrategy(new DefaultSubjectNamingStrategy()); + converter.setDynamicSchemaGenerationEnabled(false); + converter.afterPropertiesSet(); + + Message specificMessage = converter.toMessage(specificRecord, + new MutableMessageHeaders(Collections.emptyMap()), + MimeTypeUtils.parseMimeType("application/*+avro")); + SchemaReference specificRef = extractSchemaReference(MimeTypeUtils.parseMimeType( + specificMessage.getHeaders().get("contentType").toString())); + + Message genericMessage = converter.toMessage(genericRecord, + new MutableMessageHeaders(Collections.emptyMap()), + MimeTypeUtils.parseMimeType("application/*+avro")); + SchemaReference genericRef = extractSchemaReference(MimeTypeUtils.parseMimeType( + genericMessage.getHeaders().get("contentType").toString())); + + assertThat(specificRef).isEqualTo(genericRef); + assertThat(genericRef.getVersion()).isEqualTo(1); + } + + public void testOriginalContentTypeHeaderOnly() throws Exception { + User specificRecord = new User(); + specificRecord.setName("joe"); + Schema v1 = new Schema.Parser().parse(AvroMessageConverterSerializationTests.class + .getClassLoader().getResourceAsStream("schemas/user.avsc")); + GenericRecord genericRecord = new GenericData.Record(v1); + genericRecord.put("name", "joe"); + SchemaRegistryClient client = new DefaultSchemaRegistryClient(restTemplateBuilder); + client.register("user", "avro", v1.toString()); + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaRegistryClientMessageConverter converter = new AvroSchemaRegistryClientMessageConverter( + client, new NoOpCacheManager(), manager); + converter.setDynamicSchemaGenerationEnabled(false); + converter.afterPropertiesSet(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DatumWriter writer = new SpecificDatumWriter<>(User.class); + Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); + writer.write(specificRecord, encoder); + encoder.flush(); + Message source = MessageBuilder.withPayload(baos.toByteArray()) + .setHeader(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_OCTET_STREAM) + .build(); + Object converted = converter.fromMessage(source, User.class); + assertThat(converted).isNotNull(); + assertThat(specificRecord.getName().toString()) + .isEqualTo(((User) converted).getName().toString()); + } + + + private SchemaReference extractSchemaReference(MimeType mimeType) { + SchemaReference schemaReference = null; + Matcher schemaMatcher = this.versionedSchema.matcher(mimeType.toString()); + if (schemaMatcher.find()) { + String subject = schemaMatcher.group(1); + Integer version = Integer.parseInt(schemaMatcher.group(2)); + schemaReference = new SchemaReference(subject, version, + AvroSchemaRegistryClientMessageConverter.AVRO_FORMAT); + } + return schemaReference; + } + + @SpringBootApplication + @EnableSchemaRegistryServer + public static class ServerApplication { + public static void main(String[] args) { + SpringApplication.run(ServerApplication.class, args); + } + } + + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/serialization/AvroSchemaRegistryClientMessageConverterTests.java b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/serialization/AvroSchemaRegistryClientMessageConverterTests.java new file mode 100644 index 000000000..1d3936dd9 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/java/org/springframework/cloud/stream/schema/serialization/AvroSchemaRegistryClientMessageConverterTests.java @@ -0,0 +1,214 @@ +/* + * Copyright 2016-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.serialization; + +import java.io.IOException; +import java.util.UUID; + +import example.avro.Command; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; +import org.springframework.boot.web.servlet.server.ServletWebServerFactory; +import org.springframework.cache.CacheManager; +import org.springframework.cache.support.NoOpCache; +import org.springframework.cache.support.NoOpCacheManager; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.cloud.stream.schema.avro.User1; +import org.springframework.cloud.stream.schema.registry.EnableSchemaRegistryServer; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaMessageConverter; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaRegistryClientMessageConverter; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManager; +import org.springframework.cloud.stream.schema.registry.avro.AvroSchemaServiceManagerImpl; +import org.springframework.cloud.stream.schema.registry.client.DefaultSchemaRegistryClient; +import org.springframework.cloud.stream.schema.registry.client.EnableSchemaRegistryClient; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.util.MimeType; +import org.springframework.web.client.RestTemplate; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.cloud.stream.schema.serialization.AvroMessageConverterSerializationTests.notification; + +/** + * @author Marius Bogoevici + * @author Oleg Zhurakousky + * @author Sercan Karaoglu + * @author James Gee + * @author Christian Tzolov + */ +public class AvroSchemaRegistryClientMessageConverterTests { + + + private ConfigurableApplicationContext schemaRegistryServerContext; + private RestTemplateBuilder restTemplateBuilder; + + @BeforeEach + public void setup() { + this.schemaRegistryServerContext = SpringApplication.run( + ServerApplication.class, "--spring.main.allow-bean-definition-overriding=true"); + this.restTemplateBuilder = this.schemaRegistryServerContext.getBean(RestTemplateBuilder.class); + } + + @AfterEach + public void tearDown() { + this.schemaRegistryServerContext.close(); + } + + @Test + public void testSendMessage() throws Exception { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(AvroSourceApplication.class)) + .web(WebApplicationType.NONE).run("--server.port=0", + "--spring.jmx.enabled=false", + "--spring.cloud.stream.bindings.myBinding-out-0.contentType=application/*+avro", + "--spring.cloud.schema.avro.dynamicSchemaGenerationEnabled=true")) { + StreamBridge streamBridge = context.getBean(StreamBridge.class); + + User1 user1 = new User1(); + user1.setFavoriteColor("foo" + UUID.randomUUID()); + user1.setName("foo" + UUID.randomUUID()); + + streamBridge.send("myBinding-out-0", user1); + + OutputDestination output = context.getBean(OutputDestination.class); + Message result = output.receive(); + + assertThat(result).isNotNull(); + + final MessageConverter userMessageConverter = (MessageConverter) context.getBean("userMessageConverter"); + final User1 receivedUser1 = (User1) userMessageConverter.fromMessage(result, User1.class); + + assertThat(receivedUser1).isNotNull(); + } + } + + @Test + public void testSchemaImportConfiguration() throws Exception { + + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(AvroSourceApplication.class)) + .web(WebApplicationType.NONE).run("--server.port=0", "--spring.jmx.enabled=false", + "--spring.cloud.schema.avro.dynamicSchemaGenerationEnabled=true", + "--spring.cloud.stream.bindings.foo-out-0.contentType=application/*+avro", + "--spring.cloud.stream.bindings.output.destination=test", + "--spring.cloud.stream.bindings.schema-registry-client.endpoint=http://localhost:8990", + "--spring.cloud.schema.avro.schema-locations=classpath:schemas/Command.avsc", + "--spring.cloud.schema.avro.schema-imports=classpath:schemas/imports/Sms.avsc," + + " classpath:schemas/imports/Email.avsc, classpath:schemas/imports/PushNotification.avsc")) { + StreamBridge streamBridge = context.getBean(StreamBridge.class); + + final Command notification = notification(); + streamBridge.send("foo-out-0", notification); + + OutputDestination output = context.getBean(OutputDestination.class); + Message result = output.receive(); + + final MessageConverter cmdConverter = (MessageConverter) context.getBean("userMessageConverter"); + final Command command = (Command) cmdConverter.fromMessage(result, Command.class); + + assertThat(command).isNotNull(); + } + } + + @Test + public void testNoCacheConfiguration() { + ConfigurableApplicationContext sourceContext = SpringApplication + .run(NoCacheConfiguration.class, "--spring.main.web-environment=false"); + AvroSchemaRegistryClientMessageConverter converter = sourceContext + .getBean(AvroSchemaRegistryClientMessageConverter.class); + DirectFieldAccessor accessor = new DirectFieldAccessor(converter); + assertThat(accessor.getPropertyValue("cacheManager")).isInstanceOf(NoOpCacheManager.class); + + } + + @Test + public void testNamedCacheIsRequested() { + CacheManager mockCache = Mockito.mock(CacheManager.class); + when(mockCache.getCache(any())).thenReturn(new NoOpCache("")); + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaRegistryClientMessageConverter converter = new AvroSchemaRegistryClientMessageConverter( + new DefaultSchemaRegistryClient(restTemplateBuilder), mockCache, manager); + ReflectionTestUtils.invokeMethod(converter, "getCache", "TEST_CACHE"); + verify(mockCache).getCache("TEST_CACHE"); + } + + @EnableAutoConfiguration + @EnableSchemaRegistryClient + public static class AvroSourceApplication { + + private Resource schemaLocation; + + @Bean + public MessageConverter userMessageConverter() throws IOException { + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter( + MimeType.valueOf("application/avro"), manager); + if (this.schemaLocation != null) { + avroSchemaMessageConverter.setSchemaLocation(this.schemaLocation); + } + return avroSchemaMessageConverter; + } + + } + + @Configuration + public static class NoCacheConfiguration { + + @Bean + AvroSchemaRegistryClientMessageConverter avroSchemaRegistryClientMessageConverter() { + AvroSchemaServiceManager manager = new AvroSchemaServiceManagerImpl(); + return new AvroSchemaRegistryClientMessageConverter( + new DefaultSchemaRegistryClient(new RestTemplate()), new NoOpCacheManager(), manager); + } + + @Bean + ServletWebServerFactory servletWebServerFactory() { + return new TomcatServletWebServerFactory(); + } + + } + + @SpringBootApplication + @EnableSchemaRegistryServer + public static class ServerApplication { + public static void main(String[] args) { + SpringApplication.run(AvroMessageConverterSerializationTests.ServerApplication.class, args); + } + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/Command.avsc b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/Command.avsc new file mode 100644 index 000000000..3cfc1d9ce --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/Command.avsc @@ -0,0 +1,19 @@ +{ + "namespace":"example.avro", + "name":"Command", + "type":"record", + "fields":[ + { + "name":"type", + "type":"string" + }, + { + "name":"correlationId", + "type":"string" + }, + { + "name":"payload", + "type":["Sms", "Email", "PushNotification"] + } + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/Email.avsc b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/Email.avsc new file mode 100644 index 000000000..2b24fc236 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/Email.avsc @@ -0,0 +1,19 @@ +{ + "namespace":"example.avro", + "name": "Email", + "type": "record", + "fields":[ + { + "name":"addressTo", + "type":"string" + }, + { + "name":"title", + "type":"string" + }, + { + "name":"text", + "type":"string" + } + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/PushNotification.avsc b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/PushNotification.avsc new file mode 100644 index 000000000..afded1f20 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/PushNotification.avsc @@ -0,0 +1,15 @@ +{ + "namespace":"example.avro", + "name": "PushNotification", + "type": "record", + "fields":[ + { + "name":"arn", + "type":"string" + }, + { + "name":"text", + "type":"string" + } + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/Sms.avsc b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/Sms.avsc new file mode 100644 index 000000000..90b7ced8c --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/imports/Sms.avsc @@ -0,0 +1,14 @@ +{ + "namespace":"example.avro", + "name": "Sms", + "type": "record", + "fields":[ + { + "name":"phoneNumber", + "type":"string" + },{ + "name":"text", + "type":"string" + } + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/status.avsc b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/status.avsc new file mode 100644 index 000000000..972a80c08 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/status.avsc @@ -0,0 +1,10 @@ +{ + "namespace":"org.springframework.cloud.stream.samples", + "name": "Status", + "type" : "record", + "fields": [ + {"name": "id", "type": "string"}, + {"name": "text", "type": "string"}, + {"name": "timestamp", "type": "long"} + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user.avsc b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user.avsc new file mode 100644 index 000000000..662029a22 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user.avsc @@ -0,0 +1,10 @@ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favoriteNumber", "type": ["int", "null"]}, + {"name": "favoriteColor", "type": ["string", "null"]} + + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_multiple_records.schema b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_multiple_records.schema new file mode 100644 index 000000000..427176af1 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_multiple_records.schema @@ -0,0 +1,22 @@ +[ + {"namespace": "org.springframework.cloud.schema.avro", + "type": "record", + "name": "User1", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favoriteNumber", "type": ["int", "null"]}, + {"name": "favoriteColor", "type": ["string", "null"]} + + ] + }, + {"namespace": "org.springframework.cloud.schema.avro.v2", + "type": "record", + "name": "User1", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favoriteNumber", "type": ["int", "null"]}, + {"name": "favoriteColor", "type": ["string", "null"]}, + {"name": "favoritePlace", "type": ["string","null"], "default" : "NYC"} + ] + } +] diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_v1.schema b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_v1.schema new file mode 100644 index 000000000..40ff89c1c --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_v1.schema @@ -0,0 +1,10 @@ +{"namespace": "org.springframework.cloud.schema.avro", + "type": "record", + "name": "User1", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favoriteNumber", "type": ["int", "null"]}, + {"name": "favoriteColor", "type": ["string", "null"]} + + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_v2.schema b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_v2.schema new file mode 100644 index 000000000..c07c61058 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user1_v2.schema @@ -0,0 +1,10 @@ +{"namespace": "org.springframework.cloud.schema.avro.v2", + "type": "record", + "name": "User1", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favoriteNumber", "type": ["int", "null"]}, + {"name": "favoriteColor", "type": ["string", "null"]}, + {"name": "favoritePlace", "type": ["string","null"], "default" : "NYC"} + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user_v2.avsc b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user_v2.avsc new file mode 100644 index 000000000..02f8ac952 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/user_v2.avsc @@ -0,0 +1,10 @@ +{"namespace": "example.avro.v2", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favoriteNumber", "type": ["int", "null"]}, + {"name": "favoriteColor", "type": ["string", "null"]}, + {"name": "favoritePlace", "type": ["string","null"], "default" : "NYC"} + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/users_v1.schema b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/users_v1.schema new file mode 100644 index 000000000..662029a22 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/users_v1.schema @@ -0,0 +1,10 @@ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favoriteNumber", "type": ["int", "null"]}, + {"name": "favoriteColor", "type": ["string", "null"]} + + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/users_v2.schema b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/users_v2.schema new file mode 100644 index 000000000..1cb8c4130 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-client/src/test/resources/schemas/users_v2.schema @@ -0,0 +1,10 @@ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favoriteNumber", "type": ["int", "null"]}, + {"name": "favoriteColor", "type": ["string", "null"]}, + {"name": "favoritePlace", "type": ["string","null"], "default" : "NYC"} + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/.jdk8 b/schema-registry/spring-cloud-stream-schema-registry-core/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/pom.xml b/schema-registry/spring-cloud-stream-schema-registry-core/pom.xml new file mode 100644 index 000000000..538bcde0e --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/pom.xml @@ -0,0 +1,41 @@ + + + 4.0.0 + spring-cloud-stream-schema-registry-core + + + spring-cloud-stream-schema-registry + org.springframework.cloud + 4.0.0-SNAPSHOT + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.boot + spring-boot-starter-data-jpa + + + com.h2database + h2 + + + org.apache.avro + avro + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/EnableSchemaRegistryServer.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/EnableSchemaRegistryServer.java new file mode 100644 index 000000000..933c01275 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/EnableSchemaRegistryServer.java @@ -0,0 +1,39 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.cloud.stream.schema.registry.config.SchemaServerConfiguration; +import org.springframework.context.annotation.Import; + +/** + * Enables the schema registry server enpoints. + * + * @author Vinicius Carvalho + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Import(SchemaServerConfiguration.class) +public @interface EnableSchemaRegistryServer { + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/config/SchemaServerConfiguration.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/config/SchemaServerConfiguration.java new file mode 100644 index 000000000..92e80abbc --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/config/SchemaServerConfiguration.java @@ -0,0 +1,64 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.config; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.boot.autoconfigure.domain.EntityScanPackages; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.stream.schema.registry.controllers.ServerController; +import org.springframework.cloud.stream.schema.registry.model.Schema; +import org.springframework.cloud.stream.schema.registry.repository.SchemaRepository; +import org.springframework.cloud.stream.schema.registry.support.AvroSchemaValidator; +import org.springframework.cloud.stream.schema.registry.support.SchemaValidator; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +/** + * @author Vinicius Carvalho + * @author Soby Chacko + */ +@Configuration +@EnableJpaRepositories(basePackageClasses = SchemaRepository.class) +@EnableConfigurationProperties(SchemaServerProperties.class) +@Import(ServerController.class) +public class SchemaServerConfiguration { + + @Bean + public static BeanFactoryPostProcessor entityScanPackagesPostProcessor() { + return beanFactory -> { + if (beanFactory instanceof BeanDefinitionRegistry) { + EntityScanPackages.register((BeanDefinitionRegistry) beanFactory, + Collections.singletonList(Schema.class.getPackage().getName())); + } + }; + } + + @Bean + public Map schemaValidators() { + Map validatorMap = new HashMap<>(); + validatorMap.put("avro", new AvroSchemaValidator()); + return validatorMap; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/config/SchemaServerProperties.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/config/SchemaServerProperties.java new file mode 100644 index 000000000..743f982b6 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/config/SchemaServerProperties.java @@ -0,0 +1,57 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * @author Vinicius Carvalho + * @author Ilayaperumal Gopinathan + * @author Christian Tzolov + */ +@ConfigurationProperties("spring.cloud.stream.schema.server") +public class SchemaServerProperties { + + /** + * Prefix for configuration resource paths (default is empty). Useful when embedding + * in another application when you don't want to change the context path or servlet + * path. + */ + private String path; + + /** + * Boolean flag to enable/disable schema deletion. + */ + private boolean allowSchemaDeletion; + + public String getPath() { + return this.path; + } + + public void setPath(String path) { + this.path = path; + } + + public boolean isAllowSchemaDeletion() { + return this.allowSchemaDeletion; + } + + public void setAllowSchemaDeletion(boolean allowSchemaDeletion) { + this.allowSchemaDeletion = allowSchemaDeletion; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/controllers/ServerController.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/controllers/ServerController.java new file mode 100644 index 000000000..ebf0d5b32 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/controllers/ServerController.java @@ -0,0 +1,274 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.controllers; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.springframework.cloud.stream.schema.registry.config.SchemaServerProperties; +import org.springframework.cloud.stream.schema.registry.model.Schema; +import org.springframework.cloud.stream.schema.registry.repository.SchemaRepository; +import org.springframework.cloud.stream.schema.registry.support.InvalidSchemaException; +import org.springframework.cloud.stream.schema.registry.support.SchemaDeletionNotAllowedException; +import org.springframework.cloud.stream.schema.registry.support.SchemaNotFoundException; +import org.springframework.cloud.stream.schema.registry.support.SchemaValidator; +import org.springframework.cloud.stream.schema.registry.support.UnsupportedFormatException; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.lang.NonNull; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.util.UriComponentsBuilder; + +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + +/** + * @author Vinicius Carvalho + * @author Ilayaperumal Gopinathan + * @author Jeff Maxwell + * @author Christian Tzolov + */ +@RestController +@RequestMapping(path = "${spring.cloud.stream.schema.server.path:}") +public class ServerController { + + private final SchemaRepository repository; + + private final Map validators; + + private final SchemaServerProperties schemaServerProperties; + + public ServerController(SchemaRepository repository, Map validators, + SchemaServerProperties schemaServerProperties) { + Assert.notNull(repository, "cannot be null"); + Assert.notEmpty(validators, "cannot be empty"); + this.repository = repository; + this.validators = validators; + this.schemaServerProperties = schemaServerProperties; + } + + @RequestMapping(method = RequestMethod.POST, path = "/", consumes = "application/json", produces = "application/json") + public synchronized ResponseEntity register(@RequestBody Schema schema, UriComponentsBuilder builder) { + + SchemaValidator validator = this.validators.get(schema.getFormat()); + + if (validator == null) { + throw new UnsupportedFormatException(String.format("Invalid format, supported types are: %s", + StringUtils.collectionToCommaDelimitedString(this.validators.keySet()))); + } + + validator.validate(schema.getDefinition()); + + Schema result; + List registeredEntities = + this.repository.findBySubjectAndFormatOrderByVersion(schema.getSubject(), schema.getFormat()); + if (registeredEntities.isEmpty()) { + schema.setVersion(1); + result = this.repository.save(schema); + } + else { + result = validator.match(registeredEntities, schema.getDefinition()); + if (result == null) { + schema.setVersion(registeredEntities.get(registeredEntities.size() - 1).getVersion() + 1); + result = this.repository.save(schema); + } + + } + + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.LOCATION, builder.path("/{subject}/{format}/v{version}") + .buildAndExpand(result.getSubject(), result.getFormat(), result.getVersion()) + .toString()); + ResponseEntity response = new ResponseEntity<>(result, headers, HttpStatus.CREATED); + + return response; + + } + + @RequestMapping(method = RequestMethod.GET, produces = "application/json", path = "/{subject}/{format}/v{version}") + public ResponseEntity findOne(@PathVariable("subject") String subject, + @PathVariable("format") String format, + @PathVariable("version") Integer version) { + Schema schema = this.repository.findOneBySubjectAndFormatAndVersion(subject, format, version); + if (schema == null) { + throw new SchemaNotFoundException( + String.format("Could not find Schema by subject: %s, format: %s, version %s", + subject, format, version)); + } + return new ResponseEntity<>(schema, HttpStatus.OK); + } + + @RequestMapping(method = RequestMethod.GET, produces = "application/json", path = "/schemas/{id}") + public ResponseEntity findOne(@PathVariable("id") Integer id) { + Optional schema = this.repository.findById(id); + if (!schema.isPresent()) { + throw new SchemaNotFoundException(String.format("Could not find Schema by id: %s", id)); + } + return new ResponseEntity<>(schema.get(), HttpStatus.OK); + } + + /** + *

    + * Find by {@link Schema#getSubject() subject} and {@link Schema#getFormat() format}. + * + * @param subject the {@link Schema#getSubject() subject}, must not be + * {@literal null}. + * @param format the {@link Schema#getFormat() format}, must not be {@literal null}. + * @return An {@link HttpStatus#OK} response populated with the list of {@link Schema + * Schemas}, in ascending order by {@link Schema#getVersion() version}, that matched + * the supplied {@link Schema#getSubject() subject} and {@link Schema#getFormat() + * format}. + * @deprecated use {@link #findBySubjectAndFormat(String, String)} + * @see GH-1760 + */ + @Deprecated + public ResponseEntity> findBySubjectAndVersion(@PathVariable("subject") String subject, + @PathVariable("format") String format) { + return findBySubjectAndFormatOrderByVersionAsc(subject, format); + } + + /** + * Find by {@link Schema#getSubject() subject} and {@link Schema#getFormat() format}. + * + * @param subject the {@link Schema#getSubject() subject}, must not be + * {@literal null}. + * @param format the {@link Schema#getFormat() format}, must not be {@literal null}. + * @return An {@link HttpStatus#OK} response populated with the list of {@link Schema + * Schemas}, in ascending order by {@link Schema#getVersion() version}, that matched + * the supplied {@link Schema#getSubject() subject} and {@link Schema#getFormat() + * format}. + * + * @since 3.0.0 + */ + @GetMapping(produces = APPLICATION_JSON_VALUE, path = "/{subject}/{format}") + @NonNull + public ResponseEntity> findBySubjectAndFormat(@NonNull @PathVariable("subject") final String subject, + @NonNull @PathVariable("format") final String format) { + return findBySubjectAndFormatOrderByVersionAsc(subject, format); + } + + @RequestMapping(value = "/{subject}/{format}/v{version}", method = RequestMethod.DELETE) + public void delete(@PathVariable("subject") String subject, + @PathVariable("format") String format, + @PathVariable("version") Integer version) { + if (this.schemaServerProperties.isAllowSchemaDeletion()) { + Schema schema = this.repository.findOneBySubjectAndFormatAndVersion(subject, format, version); + if (schema == null) { + throw new SchemaNotFoundException( + String.format("Could not find Schema by subject: %s, format: %s, version %s", + subject, format, version)); + } + deleteSchema(schema); + } + else { + throw new SchemaDeletionNotAllowedException(String.format("Not permitted deletion of Schema by " + + "subject: %s, format: %s, version %s", subject, format, version)); + } + } + + @RequestMapping(value = "/schemas/{id}", method = RequestMethod.DELETE) + public void delete(@PathVariable("id") Integer id) { + if (this.schemaServerProperties.isAllowSchemaDeletion()) { + Optional schema = this.repository.findById(id); + if (!schema.isPresent()) { + throw new SchemaNotFoundException(String.format("Could not find Schema by id: %s", id)); + } + deleteSchema(schema.get()); + } + else { + throw new SchemaDeletionNotAllowedException(String.format("Not permitted deletion of Schema by id: %s", id)); + } + } + + @RequestMapping(value = "/{subject}", method = RequestMethod.DELETE) + public void delete(@PathVariable("subject") String subject) { + if (this.schemaServerProperties.isAllowSchemaDeletion()) { + for (Schema schema : this.repository.findAll()) { + if (schema.getSubject().equals(subject)) { + deleteSchema(schema); + } + } + } + else { + throw new SchemaDeletionNotAllowedException(String.format("Not permitted deletion of Schema by " + + "subject: %s", subject)); + } + + } + + @NonNull + public final ResponseEntity> findBySubjectAndFormatOrderByVersionAsc(@NonNull final String subject, + @NonNull final String format) { + List schemas = this.repository.findBySubjectAndFormatOrderByVersion(subject, format); + if (schemas.isEmpty()) { + throw new SchemaNotFoundException( + String.format("No schemas found for subject %s and format %s", subject, format)); + } + return new ResponseEntity<>(schemas, HttpStatus.OK); + } + + private void deleteSchema(Schema schema) { + if (schema == null) { + throw new SchemaNotFoundException("Could not find Schema"); + } + this.repository.delete(schema); + } + + @ExceptionHandler(UnsupportedFormatException.class) + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ResponseBody + public String onUnsupportedFormat(UnsupportedFormatException e) { + return errorMessage("Format not supported", e); + } + + @ExceptionHandler(InvalidSchemaException.class) + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ResponseBody + public String onInvalidSchema(InvalidSchemaException e) { + return errorMessage("Invalid Schema", e); + } + + @ExceptionHandler(SchemaNotFoundException.class) + @ResponseStatus(HttpStatus.NOT_FOUND) + @ResponseBody + public String schemaNotFound(SchemaNotFoundException ex) { + return errorMessage("Schema not found", ex); + } + + @ExceptionHandler(SchemaDeletionNotAllowedException.class) + @ResponseStatus(HttpStatus.METHOD_NOT_ALLOWED) + @ResponseBody + public String schemaDeletionNotPermitted(SchemaDeletionNotAllowedException ex) { + return errorMessage("Schema deletion is not permitted", ex); + } + + private String errorMessage(String prefix, Throwable e) { + return prefix + (StringUtils.hasText(e.getMessage()) ? ": " + e.getMessage() : ""); + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/model/Compatibility.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/model/Compatibility.java new file mode 100644 index 000000000..ebf0adfc2 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/model/Compatibility.java @@ -0,0 +1,44 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.model; + +/** + * @author Vinicius Carvalho + */ +public enum Compatibility { + + /** + * Backward compatibiltity. + */ + BACKWARD, + + /** + * Forward compatibility. + */ + FORWARD, + + /** + * Full compatibility. + */ + FULL, + + /** + * Lack of compatibility. + */ + INCOMPATIBLE; + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/model/Schema.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/model/Schema.java new file mode 100644 index 000000000..d8b10546b --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/model/Schema.java @@ -0,0 +1,93 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Lob; +import jakarta.persistence.Table; + +/** + * @author Vinicius Carvalho + * + * Represents a persisted schema entity. + */ +@Entity +@Table(name = "SCHEMA_REPOSITORY") +public class Schema { + + @Id + @GeneratedValue + @Column(name = "ID") + private Integer id; + + @Column(name = "VERSION", nullable = false) + private Integer version; + + @Column(name = "SUBJECT", nullable = false) + private String subject; + + @Column(name = "FORMAT", nullable = false) + private String format; + + @Lob + @Column(name = "DEFINITION", nullable = false, length = 8192) + private String definition; + + public Integer getId() { + return this.id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Integer getVersion() { + return this.version; + } + + public void setVersion(Integer version) { + this.version = version; + } + + public String getSubject() { + return this.subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getFormat() { + return this.format; + } + + public void setFormat(String format) { + this.format = format; + } + + public String getDefinition() { + return this.definition; + } + + public void setDefinition(String definition) { + this.definition = definition; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/repository/SchemaRepository.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/repository/SchemaRepository.java new file mode 100644 index 000000000..7dd7b7de3 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/repository/SchemaRepository.java @@ -0,0 +1,36 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.repository; + +import java.util.List; + +import org.springframework.cloud.stream.schema.registry.model.Schema; +import org.springframework.data.repository.CrudRepository; +import org.springframework.transaction.annotation.Transactional; + +/** + * @author Vinicius Carvalho + */ +public interface SchemaRepository extends CrudRepository { + + @Transactional + List findBySubjectAndFormatOrderByVersion(String subject, String format); + + @Transactional + Schema findOneBySubjectAndFormatAndVersion(String subject, String format, Integer version); + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/AvroSchemaValidator.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/AvroSchemaValidator.java new file mode 100644 index 000000000..4ca4fd9e1 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/AvroSchemaValidator.java @@ -0,0 +1,83 @@ +/* + * Copyright 2016-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.support; + +import java.util.List; + +import org.apache.avro.SchemaParseException; + +import org.springframework.cloud.stream.schema.registry.model.Compatibility; +import org.springframework.cloud.stream.schema.registry.model.Schema; + +/** + * @author Vinicius Carvalho + * @author Christian Tzolov + */ +public class AvroSchemaValidator implements SchemaValidator { + + /** + * Unique Avro schema format identifier. + */ + public static final String AVRO_FORMAT = "avro"; + + @Override + public boolean isValid(String definition) { + boolean result = true; + try { + new org.apache.avro.Schema.Parser().parse(definition); + } + catch (SchemaParseException ex) { + result = false; + } + return result; + } + + @Override + public void validate(String definition) { + try { + new org.apache.avro.Schema.Parser().parse(definition); + } + catch (SchemaParseException ex) { + throw new InvalidSchemaException((ex.getMessage())); + } + } + + @Override + public Compatibility compatibilityCheck(String source, String other) { + return null; + } + + @Override + public Schema match(List schemas, String definition) { + Schema result = null; + org.apache.avro.Schema source = new org.apache.avro.Schema.Parser().parse(definition); + for (Schema s : schemas) { + org.apache.avro.Schema target = new org.apache.avro.Schema.Parser().parse(s.getDefinition()); + if (target.equals(source)) { + result = s; + break; + } + } + return result; + } + + @Override + public String getFormat() { + return AVRO_FORMAT; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/InvalidSchemaException.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/InvalidSchemaException.java new file mode 100644 index 000000000..cb404fc05 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/InvalidSchemaException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.support; + +/** + * @author Vinicius Carvalho + */ +public class InvalidSchemaException extends RuntimeException { + + public InvalidSchemaException(String message) { + super(message); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaDeletionNotAllowedException.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaDeletionNotAllowedException.java new file mode 100644 index 000000000..4f740cffb --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaDeletionNotAllowedException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.support; + +/** + * @author Ilayaperumal Gopinathan + */ +public class SchemaDeletionNotAllowedException extends RuntimeException { + + public SchemaDeletionNotAllowedException(String message) { + super(message); + } + + public SchemaDeletionNotAllowedException() { + super("Schema Deletion Not Allowed"); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaNotFoundException.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaNotFoundException.java new file mode 100644 index 000000000..641d29b64 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaNotFoundException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.support; + +/** + * @author Vinicius Carvalho + */ +public class SchemaNotFoundException extends RuntimeException { + + public SchemaNotFoundException(String message) { + super(message); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaValidator.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaValidator.java new file mode 100644 index 000000000..e75735b43 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/SchemaValidator.java @@ -0,0 +1,70 @@ +/* + * Copyright 2016-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.support; + +import java.util.List; + +import org.springframework.cloud.stream.schema.registry.model.Compatibility; +import org.springframework.cloud.stream.schema.registry.model.Schema; + +/** + * @author Vinicius Carvalho + * @author Christian Tzolov + * + * Provides utility methods to validate, check compatibility and match schemas of + * different implementations + */ +public interface SchemaValidator { + + /** + * Verifies if a definition is a valid schema. + * @param definition - The textual representation of the schema file + * @return true if valid, false otherwise + */ + boolean isValid(String definition); + + /** + * Validates a schema definition and throws an {@link InvalidSchemaException} when the schema is invalid. + * The exception is expected to have the violation description. + * @param definition - The textual representation of the schema file + */ + default void validate(String definition) { + if (!this.isValid(definition)) { + throw new InvalidSchemaException("Invalid Schema"); + } + } + + /** + * Checks for compatibility between two schemas @see Compatibility class for types + * This method may not be supported for certain formats. + * @param source - The textual representation of the schema to tested + * @param other - The textual representation of the other schema to tested + * @return {@link Compatibility} + */ + Compatibility compatibilityCheck(String source, String other); + + /** + * Return the Schema that is represented by the definition. + * @param schemas List of schemas to be tested + * @param definition Textual representation of the schema + * @return A full Schema object with identifier and subject properties + */ + Schema match(List schemas, String definition); + + String getFormat(); + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/UnsupportedFormatException.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/UnsupportedFormatException.java new file mode 100644 index 000000000..7e99d7e75 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/java/org/springframework/cloud/stream/schema/registry/support/UnsupportedFormatException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.support; + +/** + * @author Vinicius Carvalho + */ +public class UnsupportedFormatException extends RuntimeException { + + public UnsupportedFormatException(String message) { + super(message); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/main/resources/application.yml b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/resources/application.yml new file mode 100644 index 000000000..a5aad28b1 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/main/resources/application.yml @@ -0,0 +1,6 @@ +spring: + application: + name: SchemaRegistryServer +server: + port: 8990 + diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/AbstractServerControllerTest.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/AbstractServerControllerTest.java new file mode 100644 index 000000000..fb63d6982 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/AbstractServerControllerTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.entityScanning; + +import org.junit.jupiter.api.BeforeEach; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.stream.schema.registry.config.SchemaServerProperties; +import org.springframework.cloud.stream.schema.registry.model.Schema; +import org.springframework.cloud.stream.schema.registry.repository.SchemaRepository; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; + + + + +/** + * @author Christian Tzolov + */ +public abstract class AbstractServerControllerTest { + + protected MockMvc mockMvc; + + @Autowired + protected SchemaRepository schemaRepository; + + @Autowired + protected SchemaServerProperties schemaServerProperties; + + @Autowired + private WebApplicationContext wac; + + @BeforeEach + public void setupMocks() { + this.mockMvc = MockMvcBuilders.webAppContextSetup(wac) + .defaultRequest(get("/").accept(MediaType.APPLICATION_JSON)).build(); + Schema schema = new Schema(); + schema.setSubject("test667"); + schema.setVersion(667); + schema.setFormat("format"); + schema.setDefinition("Test Schema Definition"); + schemaRepository.save(schema); + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/EntityScanningTests.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/EntityScanningTests.java new file mode 100644 index 000000000..0bbb70fe7 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/EntityScanningTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.entityScanning; + + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.cloud.stream.schema.registry.EnableSchemaRegistryServer; +import org.springframework.context.ConfigurableApplicationContext; + +/** + * @author Marius Bogoevici + */ +public class EntityScanningTests { + + @Test + public void testApplicationWithEmbeddedSchemaRegistryServerOutsideOfRootPackage() + throws Exception { + final ConfigurableApplicationContext context = SpringApplication + .run(CustomApplicationEmbeddingSchemaServer.class, "--server.port=0"); + context.close(); + } + + @EnableAutoConfiguration + @EnableSchemaRegistryServer + public static class CustomApplicationEmbeddingSchemaServer { + + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/EntityScanningTestsWithEntityScan.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/EntityScanningTestsWithEntityScan.java new file mode 100644 index 000000000..4f9cdae20 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/EntityScanningTestsWithEntityScan.java @@ -0,0 +1,47 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.entityScanning; + + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.cloud.stream.schema.registry.EnableSchemaRegistryServer; +import org.springframework.context.ConfigurableApplicationContext; + +/** + * @author Marius Bogoevici + */ +public class EntityScanningTestsWithEntityScan { + + @Test + public void testApplicationWithEmbeddedSchemaRegistryServerOutsideOfRootPackage() { + final ConfigurableApplicationContext context = SpringApplication + .run(CustomApplicationEmbeddingSchemaServer.class, "--server.port=0"); + context.close(); + } + + @EnableAutoConfiguration + @EnableSchemaRegistryServer + @EntityScan(basePackages = "org.springframework.cloud.stream.schema.registry.entityScanning.domain") + public static class CustomApplicationEmbeddingSchemaServer { + + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/ServerControllerTest.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/ServerControllerTest.java new file mode 100644 index 000000000..9e60c5932 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/ServerControllerTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.entityScanning; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.stream.schema.registry.config.SchemaServerConfiguration; +import org.springframework.http.MediaType; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.CoreMatchers.containsString; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * @author Christian Tzolov + */ +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = { SchemaServerConfiguration.class }) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.ANY) +@EnableAutoConfiguration +@TestPropertySource(properties = { + "spring.cloud.stream.schema.server.path=/testpath", + "spring.cloud.stream.schema.server.allowSchemaDeletion=false" +}) +public class ServerControllerTest extends AbstractServerControllerTest { + + @Test + public void propertiesTest() { + assertThat(schemaServerProperties.getPath()).isEqualTo("/testpath"); + assertThat(schemaServerProperties.isAllowSchemaDeletion()).isEqualTo(false); + } + + @Test + public void findSchema() throws Exception { + mockMvc.perform(get(schemaServerProperties.getPath() + "/test667/format/v667") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString("Test Schema Definition"))); + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/domain/TestEntity.java b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/domain/TestEntity.java new file mode 100644 index 000000000..d17c8558e --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-core/src/test/java/org/springframework/cloud/stream/schema/registry/entityScanning/domain/TestEntity.java @@ -0,0 +1,51 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.entityScanning.domain; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; + +/** + * @author Marius Bogoevici + */ +@Entity +public class TestEntity { + + @Id + private long id; + + @Column(name = "name") + private String name; + + public long getId() { + return this.id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/.jdk8 b/schema-registry/spring-cloud-stream-schema-registry-server/.jdk8 new file mode 100644 index 000000000..e69de29bb diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/pom.xml b/schema-registry/spring-cloud-stream-schema-registry-server/pom.xml new file mode 100644 index 000000000..f35add7b6 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-server/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + spring-cloud-stream-schema-registry-server + + + spring-cloud-stream-schema-registry + org.springframework.cloud + 4.0.0-SNAPSHOT + + + + springcloud + + + + + org.springframework.cloud + spring-cloud-stream-schema-registry-core + ${project.version} + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.hsqldb + hsqldb + + + org.mariadb.jdbc + mariadb-java-client + + + org.postgresql + postgresql + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/src/main/java/org/springframework/cloud/stream/schema/registry/server/SchemaRegistryServerApplication.java b/schema-registry/spring-cloud-stream-schema-registry-server/src/main/java/org/springframework/cloud/stream/schema/registry/server/SchemaRegistryServerApplication.java new file mode 100644 index 000000000..74ba842e7 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-server/src/main/java/org/springframework/cloud/stream/schema/registry/server/SchemaRegistryServerApplication.java @@ -0,0 +1,36 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.server; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.schema.registry.EnableSchemaRegistryServer; + +/** + * @author Vinicius Carvalho + */ +// @checkstyle:off +@SpringBootApplication +@EnableSchemaRegistryServer +public class SchemaRegistryServerApplication { + + public static void main(String[] args) { + SpringApplication.run(SchemaRegistryServerApplication.class, args); + } + +} +// @checkstyle:on diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/src/main/resources/META-INF/spring.factories b/schema-registry/spring-cloud-stream-schema-registry-server/src/main/resources/META-INF/spring.factories new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-server/src/main/resources/META-INF/spring.factories @@ -0,0 +1 @@ + diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/src/main/resources/application.yml b/schema-registry/spring-cloud-stream-schema-registry-server/src/main/resources/application.yml new file mode 100644 index 000000000..a5aad28b1 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-server/src/main/resources/application.yml @@ -0,0 +1,6 @@ +spring: + application: + name: SchemaRegistryServer +server: + port: 8990 + diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/src/test/java/org/springframework/cloud/stream/schema/registry/server/SchemaRegistryServerAvroTests.java b/schema-registry/spring-cloud-stream-schema-registry-server/src/test/java/org/springframework/cloud/stream/schema/registry/server/SchemaRegistryServerAvroTests.java new file mode 100644 index 000000000..e23f8352a --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-server/src/test/java/org/springframework/cloud/stream/schema/registry/server/SchemaRegistryServerAvroTests.java @@ -0,0 +1,623 @@ +/* + * Copyright 2016-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.schema.registry.server; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.stream.Stream; + +import org.apache.avro.Schema.Parser; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.web.ServerProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.boot.web.server.Ssl; +import org.springframework.cloud.stream.schema.registry.config.SchemaServerProperties; +import org.springframework.cloud.stream.schema.registry.controllers.ServerController; +import org.springframework.cloud.stream.schema.registry.model.Schema; +import org.springframework.cloud.stream.schema.registry.support.SchemaNotFoundException; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.HttpStatusCode; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.lang.NonNull; +import org.springframework.lang.Nullable; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.StreamUtils; +import org.springframework.web.client.DefaultResponseErrorHandler; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.util.UriComponentsBuilder; + +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.fail; +import static org.springframework.test.annotation.DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD; + +/** + * @author Vinicius Carvalho + * @author Ilayaperumal Gopinathan + * @author Christian Tzolov + */ +@ExtendWith(SpringExtension.class) +// @checkstyle:off +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = "spring.main.allow-bean-definition-overriding=true") +// @checkstyle:on +@DirtiesContext(classMode = AFTER_EACH_TEST_METHOD) +public class SchemaRegistryServerAvroTests { + + private static final String AVRO_FORMAT_NAME = "avro"; + + private static final org.apache.avro.Schema AVRO_USER_AVRO_SCHEMA_V1 = new Parser() + .parse(resourceToString("classpath:/avro_user_definition_schema_v1.json")); + + private static final org.apache.avro.Schema AVRO_USER_AVRO_SCHEMA_V2 = new Parser() + .parse(resourceToString("classpath:/avro_user_definition_schema_v2.json")); + + private static final String AVRO_USER_SCHEMA_DEFAULT_NAME_STRATEGY_SUBJECT = AVRO_USER_AVRO_SCHEMA_V1.getName() + .toLowerCase(); + + + private static final String AVRO_USER_SCHEMA_QUALIFED_NAME_STRATEGY_SUBJECT = AVRO_USER_AVRO_SCHEMA_V1 + .getFullName() + .toLowerCase(); + + private static final Schema AVRO_USER_REGISTRY_SCHEMA_V1 = toSchema( + AVRO_USER_SCHEMA_DEFAULT_NAME_STRATEGY_SUBJECT, + AVRO_FORMAT_NAME, AVRO_USER_AVRO_SCHEMA_V1.toString()); + + private static final Schema AVRO_USER_REGISTRY_SCHEMA_V2 = toSchema( + AVRO_USER_SCHEMA_DEFAULT_NAME_STRATEGY_SUBJECT, + AVRO_FORMAT_NAME, AVRO_USER_AVRO_SCHEMA_V2.toString()); + + private static final Schema AAVRO_USER_REGISTRY_SCHEMA_V1_WITH_QUAL_SUBJECT = toSchema( + AVRO_USER_SCHEMA_QUALIFED_NAME_STRATEGY_SUBJECT, + AVRO_FORMAT_NAME, AVRO_USER_AVRO_SCHEMA_V1.toString()); + @Autowired + private TestRestTemplate client; + + @Autowired + private SchemaServerProperties schemaServerProperties; + + @Autowired + private ServerController serverController; + + @Autowired + private ServerProperties serverProperties; + + private URI serverControllerUri; + + @BeforeEach + public void setUp() { + + String scheme = Optional.ofNullable(this.serverProperties.getSsl()) + .filter(Ssl::isEnabled) + .map(ssl -> "https").orElse("http"); + + Integer port = this.serverProperties.getPort(); + String contextPath = this.serverProperties.getServlet().getContextPath(); + + this.serverControllerUri = UriComponentsBuilder.newInstance().scheme(scheme) + .host("localhost") + .port(port) + .path(contextPath).build().toUri(); + + this.client.getRestTemplate().setErrorHandler(new DefaultResponseErrorHandler()); + + } + + @NonNull + static Schema toSchema(String subject, String format, String definition) { + Schema schema = new Schema(); + schema.setSubject(subject); + schema.setFormat(format); + schema.setDefinition(definition); + return schema; + } + + @Test + public void testUnsupportedFormat() { + Schema schema = toSchema("spring", "boot", null); + try { + this.client.postForEntity(this.serverControllerUri, schema, Schema.class); + fail("Expects: " + HttpStatus.BAD_REQUEST + " error"); + } + catch (HttpClientErrorException.BadRequest badRequest) { + assertThat(badRequest.getMessage()).isEqualTo("400 : \"Format not supported: Invalid format, supported types are: avro\""); + } + + } + + @Test + public void testInvalidSchema() { + Schema schema = toSchema("boot", AVRO_FORMAT_NAME, "{}"); + try { + this.client.postForEntity(this.serverControllerUri, schema, Schema.class); + fail("Expects: " + HttpStatus.BAD_REQUEST + " error"); + } + catch (HttpClientErrorException.BadRequest badRequest) { + assertThat(badRequest.getMessage()).isEqualTo("400 : \"Invalid Schema: No type: {}\""); + } + + } + + @Test + public void testInvalidSchemaGh22() { + Schema schema = toSchema("boot", AVRO_FORMAT_NAME, + resourceToString("classpath:/invalid_schema.json")); + try { + this.client.postForEntity(this.serverControllerUri, schema, Schema.class); + fail("Expects: " + HttpStatus.BAD_REQUEST + " error"); + } + catch (HttpClientErrorException.BadRequest badRequest) { + assertThat(badRequest.getMessage()).isEqualTo("400 : \"Invalid Schema: \"SomeType\" is not a defined name. " + + "The type of the \"field\" field must be a defined name or a {\"type\": ...} expression.\""); + } + + } + + @Test + public void testRegister1AvroSchema() { + Schema schema = toSchema("org.springframework.cloud.stream.schema.User", AVRO_FORMAT_NAME, + resourceToString("classpath:/avro_user_definition_schema_v1.json")); + registerSchemaAndAssertSuccess(schema, 1, 1); + + } + + @Test + public void testFindByIdFound() { + + ResponseEntity registerSchemaReponse = registerSchemaAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + Schema registeredSchema = registerSchemaReponse.getBody(); + + URI findByIdUriId1 = this.serverControllerUri.resolve("/schemas/" + registeredSchema.getId()); + + ResponseEntity findByIdResponse = this.client + .getForEntity(findByIdUriId1, Schema.class); + + assertThat(findByIdResponse.getStatusCode().is2xxSuccessful()).isTrue(); + + Schema actual = findByIdResponse.getBody(); + assertSchema(registeredSchema, actual); + } + + @Test + public void testFindByIdNotFound() { + + registerSchemaAndAssertSuccess(AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + URI findByIdUriId1 = this.serverControllerUri.resolve("/schemas/" + 2); + + try { + this.client.getForEntity(findByIdUriId1, Schema.class); + fail("Expects: " + HttpStatus.NOT_FOUND + " error"); + } + catch (HttpClientErrorException.NotFound notFound) { + assertThat(notFound.getMessage()).isEqualTo("404 : \"Schema not found: Could not find Schema by id: 2\""); + } + } + + @Test + public void testUserSchemaV2() { + registerSchemasAndAssertSuccess(AVRO_USER_REGISTRY_SCHEMA_V1, AVRO_USER_REGISTRY_SCHEMA_V2); + } + + @Test + public void testIdempotentRegistration() { + + registerSchemaAndAssertSuccess(AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + registerSchemaAndAssertSuccess(AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + } + + //@Test(expected = HttpClientErrorException.NotFound.class) + public void testSchemaNotfound() { + this.client.getForEntity("http://localhost:8990/foo/avro/v42", Schema.class); + } + + @Test + public void testSchemaDeletionBySubjectFormatVersion() { + + ResponseEntity registerSchemaAndAssertSuccess = registerSchemaAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + this.schemaServerProperties.setAllowSchemaDeletion(true); + + URI subjectFormatVersionUri = this.serverControllerUri + .resolve(registerSchemaAndAssertSuccess.getHeaders().getLocation()); + + + ResponseEntity deleteResponse = this.client.exchange( + new RequestEntity<>(HttpMethod.DELETE, subjectFormatVersionUri), + Void.class); + + assertThat(deleteResponse.getStatusCode()).isEqualTo(HttpStatus.OK); + + try { + this.client.getForEntity(subjectFormatVersionUri, Schema.class); + } + catch (HttpClientErrorException.NotFound notFound) { + assertThat(notFound.getMessage()).isEqualTo("404 : \"Schema not found: Could not find Schema by " + + "subject: user, format: avro, version 1\""); + } + + } + + @Test + public void testSchemaDeletionBySubjectFormatVersionNotFound() { + + ResponseEntity registerSchemaAndAssertSuccess = registerSchemaAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + this.schemaServerProperties.setAllowSchemaDeletion(true); + + URI subjectFormatVersionUri = this.serverControllerUri + .resolve(registerSchemaAndAssertSuccess.getHeaders() + .getLocation().toString().replace("v1", "v100")); + + try { + this.client.exchange(new RequestEntity<>(HttpMethod.DELETE, subjectFormatVersionUri), Void.class); + } + catch (HttpClientErrorException.NotFound notFound) { + assertThat(notFound.getMessage()).isEqualTo("404 : \"Schema not found: Could not find Schema by " + + "subject: user, format: avro, version 100\""); + } + + } + + @Test + public void testSchemaDeletionBySubjectFormatVersionNotAllowed() { + + ResponseEntity registerSchemaAndAssertSuccess = registerSchemaAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + URI versionUri = this.serverControllerUri.resolve(registerSchemaAndAssertSuccess.getHeaders().getLocation()); + + try { + this.client.exchange(new RequestEntity<>(HttpMethod.DELETE, versionUri), Void.class); + } + catch (HttpClientErrorException.MethodNotAllowed methodNotAllowed) { + assertThat(methodNotAllowed.getMessage()).isEqualTo("405 : \"Schema deletion is not permitted: Not permitted " + + "deletion of Schema by subject: user, format: avro, version 1\""); + } + + } + + @Test + public void testSchemaDeletionById() { + + ResponseEntity registerSchemaAndAssertSuccess = registerSchemaAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + this.schemaServerProperties.setAllowSchemaDeletion(true); + this.client.delete(this.serverControllerUri + .resolve("/schemas/" + registerSchemaAndAssertSuccess.getBody().getVersion())); + + try { + this.client.getForEntity(registerSchemaAndAssertSuccess.getHeaders().getLocation(), Schema.class); + fail("Expects: " + HttpStatus.NOT_FOUND + " error"); + } + catch (HttpClientErrorException.NotFound notFound) { + assertThat(notFound.getMessage()).isEqualTo("404 : \"Schema not found: Could not find Schema by subject: " + + "user, format: avro, version 1\""); + } + + } + + @Test + public void testSchemaDeletionByIdNotFound() { + + registerSchemaAndAssertSuccess(AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + this.schemaServerProperties.setAllowSchemaDeletion(true); + + try { + this.client.exchange(new RequestEntity<>(HttpMethod.DELETE, + this.serverControllerUri.resolve("/schemas/" + 2)), Void.class); + fail("Expects: " + HttpStatus.NOT_FOUND + " error"); + } + catch (HttpClientErrorException.NotFound notFound) { + assertThat(notFound.getMessage()).isEqualTo("404 : \"Schema not found: Could not find Schema by id: 2\""); + } + + } + + @Test + public void testSchemaDeletionByIdNotAllowed() { + + ResponseEntity registerSchemaAndAssertSuccess = registerSchemaAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + URI schemaIdUri = this.serverControllerUri.resolve(this.serverControllerUri + .resolve("/schemas/" + registerSchemaAndAssertSuccess.getBody().getVersion())); + + try { + this.client.exchange(new RequestEntity<>(HttpMethod.DELETE, schemaIdUri), Void.class); + fail("Expects: " + HttpStatus.METHOD_NOT_ALLOWED + " error"); + } + catch (HttpClientErrorException.MethodNotAllowed methodNotAllowed) { + assertThat(methodNotAllowed.getMessage()).isEqualTo("405 : \"Schema deletion is not permitted: Not " + + "permitted deletion of Schema by id: 1\""); + } + + } + + @Test + public void testSchemaDeletionBySubject() { + Map>>> registerSchemaResponsesByFormatBySubject = registerSchemasAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, + AVRO_USER_REGISTRY_SCHEMA_V2, AAVRO_USER_REGISTRY_SCHEMA_V1_WITH_QUAL_SUBJECT); + + this.schemaServerProperties.setAllowSchemaDeletion(true); + + registerSchemaResponsesByFormatBySubject.forEach((subject, registerSchemaResponsesByFormat) -> { + + assertThat(registerSchemaResponsesByFormat).isNotEmpty(); + ResponseEntity deleteBySubject = this.client.exchange( + new RequestEntity<>(HttpMethod.DELETE, this.serverControllerUri + .resolve("/" + subject)), + Void.class); + + assertThat(deleteBySubject.getStatusCode()).isEqualTo(HttpStatus.OK); + + registerSchemaResponsesByFormat.forEach((format, registerSchemaResponses) -> { + + assertThat(registerSchemaResponses).isNotEmpty(); + + registerSchemaResponses.forEach(registerSchemaResponse -> { + + try { + this.client.getForEntity(registerSchemaResponse.getHeaders().getLocation(), Schema.class); + fail("Expects: " + HttpStatus.NOT_FOUND + " error"); + } + catch (HttpClientErrorException.NotFound notFound) { + //do nothing + } + }); + }); + }); + + } + + @Test + public void testSchemaDeletionBySubjectNotFound() { + + registerSchemaAndAssertSuccess(AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + this.schemaServerProperties.setAllowSchemaDeletion(true); + + ResponseEntity deleteBySubject = this.client.exchange( + new RequestEntity<>(HttpMethod.DELETE, this.serverControllerUri.resolve("/foo")), Void.class); + + assertThat(deleteBySubject.getStatusCode()).isEqualTo(HttpStatus.OK); + + } + + @Test + public void testSchemaDeletionBySubjectNotAllowed() { + + ResponseEntity registerSchemaAndAssertSuccess = registerSchemaAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, 1, 1); + + Schema schema = registerSchemaAndAssertSuccess.getBody(); + + try { + this.client.exchange(new RequestEntity<>(HttpMethod.DELETE, + this.serverControllerUri.resolve("/" + schema.getSubject())), Void.class); + fail("Expects: " + HttpStatus.METHOD_NOT_ALLOWED + " error"); + } + catch (HttpClientErrorException.MethodNotAllowed methodNotAllowed) { + assertThat(methodNotAllowed.getMessage()).isEqualTo("405 : \"Schema deletion is not permitted: " + + "Not permitted deletion of Schema by subject: user\""); + } + + } + + @Test + public void testFindSchemasBySubjectAndVersion() { + + Map>>> registerSchemaResponsesByFormatBySubject = registerSchemasAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, + AVRO_USER_REGISTRY_SCHEMA_V2); + + registerSchemaResponsesByFormatBySubject.forEach((subject, schemasByFormat) -> { + + assertThat(schemasByFormat).hasSize(1); + + schemasByFormat.forEach((format, schemas) -> { + assertThat(schemas).hasSize(2); + + final ResponseEntity> findBySubjectAndVersionResponseEntity = this.serverController + .findBySubjectAndVersion(subject, format); + + assertThat(findBySubjectAndVersionResponseEntity.getStatusCode().is2xxSuccessful()).isTrue(); + + final List schemaResponseBody = findBySubjectAndVersionResponseEntity.getBody(); + + assertThat(schemaResponseBody).zipSatisfy(schemas.stream().map(ResponseEntity::getBody) + .collect(toList()), this::assertSchema); + }); + }); + + } + + @Test + public void testFindBySubjectAndFormatOrderByVersionAscNoMatch() { + String subject = "test"; + + String format = AVRO_FORMAT_NAME; + + assertThatExceptionOfType(SchemaNotFoundException.class).isThrownBy(() -> this.serverController + .findBySubjectAndFormatOrderByVersionAsc(subject, format)) + .withMessage("No schemas found for subject %s and format %s", subject, format) + .withNoCause(); + + } + + @Test + public void testFindSchemasBySubjectAndFormat() { + + Map>>> registerSchemaResponsesByFormatBySubject = registerSchemasAndAssertSuccess( + AVRO_USER_REGISTRY_SCHEMA_V1, + AVRO_USER_REGISTRY_SCHEMA_V2); + + registerSchemaResponsesByFormatBySubject.forEach((subject, schemasByFormat) -> { + + assertThat(schemasByFormat).hasSize(1); + + schemasByFormat.forEach((format, schemas) -> { + assertThat(schemas).hasSize(2); + + ResponseEntity> findBySubjectFormatResponse = this.client.exchange( + this.serverControllerUri.resolve("/" + subject + "/" + format), HttpMethod.GET, null, + new ParameterizedTypeReference>() { + }); + + assertThat(findBySubjectFormatResponse.getStatusCode().is2xxSuccessful()).isTrue(); + + final List schemaResponseBody = findBySubjectFormatResponse.getBody(); + + assertThat(schemaResponseBody).zipSatisfy(schemas.stream().map(ResponseEntity::getBody) + .collect(toList()), this::assertSchema); + }); + }); + + } + + private Map>>> registerSchemasAndAssertSuccess( + @NonNull Schema... schemas) { + Map> versionsByFormatAndSubject = new HashMap<>(); + Map>>> result = new HashMap<>(); + int numOfSchemas = schemas.length; + int id = 0; + for (int i = 0; i < numOfSchemas; i++) { + Schema schema = schemas[i]; + id++; + String format = schema.getFormat(); + String subject = schema.getSubject(); + Integer version = versionsByFormatAndSubject + .compute(subject, + (_subject, currentValue) -> currentValue == null ? new HashMap<>() : currentValue) + .merge(format, 1, Integer::sum); + ResponseEntity registerSchemaResponse = registerSchemaAndAssertSuccess(schema, version, id); + result.compute(subject, + (_subject, currentValue) -> currentValue == null ? new HashMap<>() : currentValue) + + .compute(format, (_format, currentValue) -> { + List> value = currentValue == null ? new ArrayList<>() : currentValue; + value.add(registerSchemaResponse); + return value; + }); + } + Stream> asStream = result.entrySet().stream() + .map(Entry::getValue) + .map(Map::entrySet) + .flatMap(Collection::stream) + .map(Entry::getValue) + .flatMap(Collection::stream); + assertThat(asStream).hasSize(numOfSchemas); + return result; + + } + + @NonNull + private ResponseEntity registerSchemaAndAssertSuccess(@NonNull Schema schema, + @Nullable Integer expectedVersion, + @Nullable Integer expectedId) { + + ResponseEntity registerReponse = this.client + .postForEntity(this.serverControllerUri, schema, Schema.class); + + HttpStatusCode statusCode = registerReponse.getStatusCode(); + assertThat(statusCode.is2xxSuccessful()).isTrue(); + + Schema registeredSchema = registerReponse.getBody(); + assertSchema(schema, expectedVersion, expectedId, registeredSchema); + + HttpHeaders headers = registerReponse.getHeaders(); + assertLocation(headers, registeredSchema); + + return registerReponse; + } + + private void assertLocation(HttpHeaders headers, Schema registeredSchema) { + URI location = headers.getLocation(); + + assertThat(location).isNotNull(); + assertPersisted(location, registeredSchema); + } + + private void assertPersisted(URI location, Schema registeredSchema) { + + ResponseEntity findOneResponse = this.client.getForEntity(location, + Schema.class); + + HttpStatusCode statusCode = findOneResponse.getStatusCode(); + assertThat(statusCode.is2xxSuccessful()).isTrue(); + + Schema actual = findOneResponse.getBody(); + assertSchema(registeredSchema, registeredSchema.getVersion(), registeredSchema.getId(), actual); + + } + + private void assertSchema(@NonNull Schema expected, @NonNull Schema actual) { + + assertSchema(expected, expected.getVersion(), expected.getId(), actual); + } + + private void assertSchema(@NonNull Schema expected, Integer expectedVersion, Integer expectedId, + @NonNull Schema actual) { + + assertThat(actual).isEqualToIgnoringGivenFields(expected, "version", "id"); + if (expectedVersion != null) { + assertThat(actual.getVersion()).isEqualTo(expectedVersion); + } + if (expectedId != null) { + assertThat(actual.getId()).isEqualTo(expectedId); + } + } + + private static String resourceToString(String resourceUri) { + try { + return StreamUtils.copyToString(new DefaultResourceLoader().getResource(resourceUri) + .getInputStream(), StandardCharsets.UTF_8); + } + catch (IOException e) { + throw new IllegalStateException("Could not extract resource: " + resourceUri, e); + } + } +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/avro_user_definition_schema_v1.json b/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/avro_user_definition_schema_v1.json new file mode 100644 index 000000000..a6c917218 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/avro_user_definition_schema_v1.json @@ -0,0 +1,18 @@ +{ + "namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "favorite_number", + "type": [ + "int", + "null" + ] + } + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/avro_user_definition_schema_v2.json b/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/avro_user_definition_schema_v2.json new file mode 100644 index 000000000..f251134f1 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/avro_user_definition_schema_v2.json @@ -0,0 +1,25 @@ +{ + "namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "favorite_number", + "type": [ + "int", + "null" + ] + }, + { + "name": "favorite_color", + "type": [ + "string", + "null" + ] + } + ] +} diff --git a/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/invalid_schema.json b/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/invalid_schema.json new file mode 100644 index 000000000..cb1dceb42 --- /dev/null +++ b/schema-registry/spring-cloud-stream-schema-registry-server/src/test/resources/invalid_schema.json @@ -0,0 +1,11 @@ +{ + "type": "record", + "name": "SuperType", + "namespace": "some.namespace", + "fields": [ + { + "name": "field", + "type": "SomeType" + } + ] +}