diff --git a/settings.gradle b/settings.gradle index bc6b88f..a4a8bc9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,3 +5,4 @@ include "${rootProject.name}-cloudfoundry-connector" include "${rootProject.name}-spring-service-connector" include "${rootProject.name}-heroku-connector" include "${rootProject.name}-localconfig-connector" +include "${rootProject.name}-pcf-connector" diff --git a/spring-cloud-pcf-connector/.gitignore b/spring-cloud-pcf-connector/.gitignore new file mode 100644 index 0000000..bd39306 --- /dev/null +++ b/spring-cloud-pcf-connector/.gitignore @@ -0,0 +1,2 @@ +dependency-reduced-pom.xml +/bin diff --git a/spring-cloud-pcf-connector/README.md b/spring-cloud-pcf-connector/README.md new file mode 100644 index 0000000..6990824 --- /dev/null +++ b/spring-cloud-pcf-connector/README.md @@ -0,0 +1,3 @@ +#Spring Cloud PCF extension + +Currently supports the Hadoop service on Pivotal Cloud Foundry \ No newline at end of file diff --git a/spring-cloud-pcf-connector/build.gradle b/spring-cloud-pcf-connector/build.gradle new file mode 100644 index 0000000..93866dd --- /dev/null +++ b/spring-cloud-pcf-connector/build.gradle @@ -0,0 +1,8 @@ +description = 'Spring-Cloud Support for Pivotal CF' + +dependencies { + compile project(':spring-cloud-cloudfoundry-connector') + optional("org.apache.hadoop:hadoop-common:2.2.0") + + testCompile project(path: ':spring-cloud-cloudfoundry-connector', configuration: 'tests') +} diff --git a/spring-cloud-pcf-connector/publish-maven.gradle b/spring-cloud-pcf-connector/publish-maven.gradle new file mode 100644 index 0000000..632359e --- /dev/null +++ b/spring-cloud-pcf-connector/publish-maven.gradle @@ -0,0 +1,65 @@ +apply plugin: "maven" + +ext.optionalDeps = [] +ext.providedDeps = [] + +ext.optional = { optionalDeps << it } +ext.provided = { providedDeps << it } + +install { + repositories.mavenInstaller { + customizePom(pom, project) + } +} + +def customizePom(pom, gradleProject) { + pom.whenConfigured { generatedPom -> + // respect "optional" and "provided" dependencies + gradleProject.optionalDeps.each { dep -> + generatedPom.dependencies.findAll { it.artifactId == dep.name }*.optional = true + } + gradleProject.providedDeps.each { dep -> + generatedPom.dependencies.findAll { it.artifactId == dep.name }*.scope = "provided" + } + + // eliminate test-scoped dependencies (no need in maven central poms) + generatedPom.dependencies.removeAll { dep -> + dep.scope == "test" + } + + // Remove jackson dependencies, since we shade them in + generatedPom.dependencies.removeAll { dep -> + dep.groupId == "com.fasterxml.jackson.core" + } + + // add all items necessary for maven central publication + generatedPom.project { + name = gradleProject.description + description = gradleProject.description + url = "https://github.com/spring-projects/spring-cloud" + organization { + name = "Spring IO" + url = "http://projects.spring.io/spring-cloud" + } + licenses { + license { + name "The Apache Software License, Version 2.0" + url "http://www.apache.org/licenses/LICENSE-2.0.txt" + distribution "repo" + } + } + scm { + url = "https://github.com/spring-projects/spring-cloud" + connection = "scm:git:git://github.com/spring-projects/spring-cloud" + developerConnection = "scm:git:git://github.com/spring-projects/spring-cloud" + } + developers { + developer { + id = "ramnivas" + name = "Ramnivas Laddad" + email = "" + } + } + } + } +} \ No newline at end of file diff --git a/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/gemfire/GemfireXDServiceInfo.java b/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/gemfire/GemfireXDServiceInfo.java new file mode 100644 index 0000000..02bb1de --- /dev/null +++ b/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/gemfire/GemfireXDServiceInfo.java @@ -0,0 +1,20 @@ +package org.springframework.cloud.pcf.gemfire; + +import org.springframework.cloud.service.ServiceInfo.ServiceLabel; +import org.springframework.cloud.service.common.RelationalServiceInfo; + + +/** + * + * @author Ramnivas Laddad + * + */ +@ServiceLabel("gemfirexd") +public class GemfireXDServiceInfo extends RelationalServiceInfo { + + public static final String JDBC_URL_TYPE = "gemfirexd"; + + public GemfireXDServiceInfo(String id, String url) { + super(id, url, JDBC_URL_TYPE); + } +} diff --git a/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/hadoop/HadoopConfigurationCreator.java b/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/hadoop/HadoopConfigurationCreator.java new file mode 100644 index 0000000..7fe6c30 --- /dev/null +++ b/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/hadoop/HadoopConfigurationCreator.java @@ -0,0 +1,25 @@ +package org.springframework.cloud.pcf.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.springframework.cloud.service.AbstractServiceConnectorCreator; +import org.springframework.cloud.service.ServiceConnectorConfig; + +/** + * + * @author Ramnivas Laddad + * + */ +public class HadoopConfigurationCreator extends AbstractServiceConnectorCreator { + + @Override + public Configuration create(HadoopServiceInfo hadoopServiceInfo, ServiceConnectorConfig serviceConnectorConfig) { + Configuration configuration = new Configuration(); + configuration.set("fs.defaultFS", hadoopServiceInfo.getDefaultFS()); + configuration.set("yarn.resourcemanager.address", hadoopServiceInfo.getYarnResourceManagerAddress()); + configuration.set("yarn.resourcemanager.scheduler.address", hadoopServiceInfo.getYarnResourceManagerSchedulerAddress()); + + return configuration; + } + + +} diff --git a/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/hadoop/HadoopServiceInfo.java b/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/hadoop/HadoopServiceInfo.java new file mode 100644 index 0000000..5db42db --- /dev/null +++ b/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/hadoop/HadoopServiceInfo.java @@ -0,0 +1,42 @@ +package org.springframework.cloud.pcf.hadoop; + +import org.springframework.cloud.service.BaseServiceInfo; +import org.springframework.cloud.service.ServiceInfo; + +/** + * Hadoop service info (also considers yarn properties). + * + * @author Ramnivas Laddad + * + */ +public class HadoopServiceInfo extends BaseServiceInfo { + + private String defaultHdfsUri; + private String yarnResourceManagerAddress; + private String yarnResourceManagerSchedulerAddress; + + public HadoopServiceInfo(String id, String defaultHdfsUri, + String yarnResourceManagerAddress, String yarnResourceManagerSchedulerAddress) { + super(id); + this.defaultHdfsUri = defaultHdfsUri; + this.yarnResourceManagerAddress = yarnResourceManagerAddress; + this.yarnResourceManagerSchedulerAddress = yarnResourceManagerSchedulerAddress; + } + + + @ServiceInfo.ServiceProperty(category = "connection") + public String getDefaultFS() { + return defaultHdfsUri; + } + + @ServiceInfo.ServiceProperty(category = "connection") + public String getYarnResourceManagerAddress() { + return yarnResourceManagerAddress; + } + + @ServiceInfo.ServiceProperty(category = "connection") + public String getYarnResourceManagerSchedulerAddress() { + return yarnResourceManagerSchedulerAddress; + } + +} diff --git a/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/phd/PhdServiceInfoCreator.java b/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/phd/PhdServiceInfoCreator.java new file mode 100644 index 0000000..9672f6d --- /dev/null +++ b/spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/phd/PhdServiceInfoCreator.java @@ -0,0 +1,79 @@ +package org.springframework.cloud.pcf.phd; + +import java.util.Map; + +import org.springframework.cloud.cloudfoundry.CloudFoundryServiceInfoCreator; +import org.springframework.cloud.cloudfoundry.Tags; +import org.springframework.cloud.pcf.gemfire.GemfireXDServiceInfo; +import org.springframework.cloud.pcf.hadoop.HadoopServiceInfo; +import org.springframework.cloud.service.BaseCompositeServiceInfo; +import org.springframework.cloud.service.ServiceInfo; +import org.springframework.cloud.service.common.PostgresqlServiceInfo; +import org.springframework.cloud.service.common.RelationalServiceInfo; + +/** + * + * @author Ramnivas Laddad + * + */ +public class PhdServiceInfoCreator extends CloudFoundryServiceInfoCreator { + + public PhdServiceInfoCreator() { + super(new Tags("p-hd")); + } + + @SuppressWarnings("unchecked") + public BaseCompositeServiceInfo createServiceInfo(Map serviceData) { + String id = (String) serviceData.get("name"); + + Map credentials = (Map) serviceData.get("credentials"); + + ServiceInfo hadoopServiceInfo = createHadoopServiceInfo(id, credentials); + ServiceInfo hawqServiceInfo = createHawqServiceInfo(id, credentials); + ServiceInfo gemfirexdServiceInfo = createGemfireXDServiceInfo(id, credentials); + + return new BaseCompositeServiceInfo(id, hadoopServiceInfo, hawqServiceInfo, gemfirexdServiceInfo); + } + + @SuppressWarnings("unchecked") + private HadoopServiceInfo createHadoopServiceInfo(String id, Map credentials) { + Map hdfs = (Map) credentials.get("hdfs"); + Map hdfsConfig = (Map) hdfs.get("configuration"); + String defaultHdfsUri = (String) hdfsConfig.get("fs.defaultFS"); + + Map yarn = (Map) credentials.get("yarn"); + Map yarnConfig = (Map) yarn.get("configuration"); + String yarnResourceManagerAddress = (String) yarnConfig.get("yarn.resourcemanager.address"); + String yarnResourceManagerSchedulerAddress = (String) yarnConfig.get("yarn.resourcemanager.scheduler.address"); + + return new HadoopServiceInfo(id + "/hadoop", defaultHdfsUri, + yarnResourceManagerAddress, yarnResourceManagerSchedulerAddress); + } + + private RelationalServiceInfo createHawqServiceInfo(String id, Map credentials) { + String key = "hawq"; + + return new PostgresqlServiceInfo(id + "/" + key, extractDataSourceUri(id, key, credentials)); + } + + private RelationalServiceInfo createGemfireXDServiceInfo(String id, Map credentials) { + String key = "gemfirexd"; + + return new GemfireXDServiceInfo(id + "/" + key, extractDataSourceUri(id, key, credentials)); + } + + private String extractDataSourceUri(String id, String key, Map credentials) { + @SuppressWarnings("unchecked") + Map hawq = (Map) credentials.get(key); + + String uri = (String) hawq.get("uri"); + + String jdbcUriPrefix = "jdbc:"; + + if (uri.startsWith(jdbcUriPrefix)) { + uri.substring(jdbcUriPrefix.length()); + } + + return uri; + } +} diff --git a/spring-cloud-pcf-connector/src/main/resources/META-INF/services/org.springframework.cloud.cloudfoundry.CloudFoundryServiceInfoCreator b/spring-cloud-pcf-connector/src/main/resources/META-INF/services/org.springframework.cloud.cloudfoundry.CloudFoundryServiceInfoCreator new file mode 100644 index 0000000..304d95f --- /dev/null +++ b/spring-cloud-pcf-connector/src/main/resources/META-INF/services/org.springframework.cloud.cloudfoundry.CloudFoundryServiceInfoCreator @@ -0,0 +1 @@ +org.springframework.cloud.pcf.phd.PhdServiceInfoCreator diff --git a/spring-cloud-pcf-connector/src/main/resources/META-INF/services/org.springframework.cloud.service.ServiceConnectorCreator b/spring-cloud-pcf-connector/src/main/resources/META-INF/services/org.springframework.cloud.service.ServiceConnectorCreator new file mode 100644 index 0000000..cbfb5c0 --- /dev/null +++ b/spring-cloud-pcf-connector/src/main/resources/META-INF/services/org.springframework.cloud.service.ServiceConnectorCreator @@ -0,0 +1 @@ +org.springframework.cloud.pcf.hadoop.HadoopConfigurationCreator diff --git a/spring-cloud-pcf-connector/src/test/java/org/springframework/cloud/hadoop/HadoopConfigurationCreatorTest.java b/spring-cloud-pcf-connector/src/test/java/org/springframework/cloud/hadoop/HadoopConfigurationCreatorTest.java new file mode 100644 index 0000000..2a78595 --- /dev/null +++ b/spring-cloud-pcf-connector/src/test/java/org/springframework/cloud/hadoop/HadoopConfigurationCreatorTest.java @@ -0,0 +1,26 @@ +package org.springframework.cloud.hadoop; + +import static org.junit.Assert.*; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; +import org.springframework.cloud.pcf.hadoop.HadoopConfigurationCreator; +import org.springframework.cloud.pcf.hadoop.HadoopServiceInfo; + +/** + * + * @author Ramnivas Laddad + * + */ +public class HadoopConfigurationCreatorTest { + private HadoopConfigurationCreator creator = new HadoopConfigurationCreator(); + + @Test + public void hadoopConfigurationCreation() { + HadoopServiceInfo serviceInfo = new HadoopServiceInfo("phd-hadoop", "hdfs://hdfshost:1234", "yrm:2345", "yrsm:3456"); + Configuration configuration = creator.create(serviceInfo, null); + assertEquals("hdfs://hdfshost:1234", configuration.get("fs.defaultFS")); + assertEquals("yrm:2345", configuration.get("yarn.resourcemanager.address")); + assertEquals("yrsm:3456", configuration.get("yarn.resourcemanager.scheduler.address")); + } +} diff --git a/spring-cloud-pcf-connector/src/test/java/org/springframework/cloud/pcf/PhdServiceInfoCreatorTest.java b/spring-cloud-pcf-connector/src/test/java/org/springframework/cloud/pcf/PhdServiceInfoCreatorTest.java new file mode 100644 index 0000000..99ed062 --- /dev/null +++ b/spring-cloud-pcf-connector/src/test/java/org/springframework/cloud/pcf/PhdServiceInfoCreatorTest.java @@ -0,0 +1,142 @@ +package org.springframework.cloud.pcf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.junit.Test; +import org.springframework.cloud.cloudfoundry.AbstractCloudFoundryConnectorTest; +import org.springframework.cloud.pcf.gemfire.GemfireXDServiceInfo; +import org.springframework.cloud.pcf.hadoop.HadoopServiceInfo; +import org.springframework.cloud.service.BaseCompositeServiceInfo; +import org.springframework.cloud.service.ServiceInfo; +import org.springframework.cloud.service.common.PostgresqlServiceInfo; + +/** + * + * @author Ramnivas Laddad + * + */ +public class PhdServiceInfoCreatorTest extends AbstractCloudFoundryConnectorTest { + @Test + public void phdServiceCreation() { + String hadoopUsername = "hadoop-user"; + String hadoopHost = "hadoop-host"; + int hadoopPort = 6000; + String hadoopDirectory = "hadoop-dir"; + String yarnResourceHost = "yark-resource-host"; + int yarnResourcePort = 7000; + String yarnResourceSchedulerHost = "yarn-resource-scheduler-host"; + int yarnResourceSchedulerPort = 8000; + String yarnMapReduceDir = "yarn-map-reduce-dir"; + String yarnStagingDir = "yarn-staging-dir"; + String hawkHost = "hawk-host"; + int hawkPort = 9000; + String hawkUsername = "hawk-user"; + String hawkPassword = "hawk-pass"; + String gemHost = "gem-host"; + int gemPort = 10000; + String gemUsername = "gem-user"; + String gemPassword = "gem-pass"; + String gemWorkingDir = "gem-working-dir"; + + when(mockEnvironment.getEnvValue("VCAP_SERVICES")).thenReturn( + getServicesPayload( + getPhdServicePayload("phd-1", hadoopUsername, hadoopHost, hadoopPort, hadoopDirectory, + yarnResourceHost, yarnResourcePort, + yarnResourceSchedulerHost, yarnResourceSchedulerPort, yarnMapReduceDir, yarnStagingDir, + hawkHost, hawkPort, hawkUsername, hawkPassword, + gemHost, gemPort, gemUsername, gemPassword, gemWorkingDir))); + + List serviceInfos = testCloudConnector.getServiceInfos(); + BaseCompositeServiceInfo phdServiceInfo = (BaseCompositeServiceInfo) getServiceInfo(serviceInfos, "phd-1"); + assertNotNull(phdServiceInfo); + assertEquals(3, phdServiceInfo.getServiceInfos().size()); + + HadoopServiceInfo hadoopServiceInfo = extractServiceInfo(phdServiceInfo.getServiceInfos(), HadoopServiceInfo.class); + assertNotNull(hadoopServiceInfo); + assertHadoopServiceInfo("phd-1/hadoop", hadoopHost, hadoopPort, + yarnResourceHost, yarnResourcePort, yarnResourceSchedulerHost, yarnResourceSchedulerPort, hadoopServiceInfo); + + PostgresqlServiceInfo hawqServiceInfo = extractServiceInfo(phdServiceInfo.getServiceInfos(), PostgresqlServiceInfo.class); + assertEquals("phd-1/hawq", hawqServiceInfo.getId()); + assertNotNull(hawqServiceInfo); + + GemfireXDServiceInfo gemfireXDServiceInfo = extractServiceInfo(phdServiceInfo.getServiceInfos(), GemfireXDServiceInfo.class); + assertNotNull(gemfireXDServiceInfo); + assertEquals("phd-1/gemfirexd", gemfireXDServiceInfo.getId()); + } + + private void assertHadoopServiceInfo(String serviceId, String hadoopHost, int hadoopPort, + String yarnResourceHost, int yarnResourcePort, + String yarnResourceSchedulerHost, int yarnResourceSchedulerPort, + HadoopServiceInfo serviceInfo) { + assertEquals(serviceId, serviceInfo.getId()); + assertEquals(String.format("hdfs://%s:%s", hadoopHost, hadoopPort), serviceInfo.getDefaultFS()); + assertEquals(String.format("%s:%s", yarnResourceHost, yarnResourcePort), serviceInfo.getYarnResourceManagerAddress()); + assertEquals(String.format("%s:%s", yarnResourceSchedulerHost, yarnResourceSchedulerPort), serviceInfo.getYarnResourceManagerSchedulerAddress()); + } + + @SuppressWarnings("unchecked") + private T extractServiceInfo(List serviceInfos, Class typeToSearch) { + for (ServiceInfo serviceInfo: serviceInfos) { + if (serviceInfo.getClass().equals(typeToSearch)) { + return (T)serviceInfo; + } + } + return null; + } + + private String getPhdServicePayload(String serviceName, + String hadoopUsername, String hadoopHost, int hadoopPort, String hadoopDirectory, + String yarnResourceHost, int yarnResourcePort, + String yarnResourceSchedulerHost, int yarnResourceSchedulerPort, + String yarnMapReduceDir, String yarnStagingDir, + String hawkHost, int hawkPort, String hawkUsername, String hawkPassword, + String gemHost, int gemPort, String gemUsername, String gemPassword, String gemWorkingDir) { + return getPhdServicePayload("test-phd-info.json", serviceName, hadoopUsername, hadoopHost, hadoopPort, hadoopDirectory, + yarnResourceHost, yarnResourcePort, + yarnResourceSchedulerHost, yarnResourceSchedulerPort, + yarnMapReduceDir, yarnStagingDir, + hawkHost, hawkPort, hawkUsername, hawkPassword, + gemHost, gemPort, gemUsername, gemPassword, gemWorkingDir); + } + + private String getPhdServicePayload(String filename, String serviceName, + String hadoopUsername, String hadoopHost, int hadoopPort, String hadoopDirectory, + String yarnResourceHost, int yarnResourcePort, + String yarnResourceSchedulerHost, int yarnResourceSchedulerPort, + String yarnMapReduceDir, String yarnStagingDir, + String hawkHost, int hawkPort, String hawkUsername, String hawkPassword, + String gemHost, int gemPort, String gemUsername, String gemPassword, String gemWorkingDir) { + String payload = readTestDataFile(filename); + payload = payload.replace("$serviceName", serviceName); + + payload = payload.replace("$hadoop-username", hadoopUsername); + payload = payload.replace("$hdfs-host", hadoopHost); + payload = payload.replace("$hdfs-port", Integer.toString(hadoopPort)); + payload = payload.replace("$hdfs-directory", hadoopDirectory); + + payload = payload.replace("$yarn-resource-host", yarnResourceHost); + payload = payload.replace("$yarn-resource-port", Integer.toString(yarnResourcePort)); + payload = payload.replace("$yarn-resource-scheduler-host", yarnResourceSchedulerHost); + payload = payload.replace("$yarn-resource-scheduler-port", Integer.toString(yarnResourceSchedulerPort)); + payload = payload.replace("$yarn-map-reduce-dir", yarnMapReduceDir); + payload = payload.replace("$yarn-staging-dir", yarnStagingDir); + + payload = payload.replace("$hawk-host", hawkHost); + payload = payload.replace("$hawk-port", Integer.toString(hawkPort)); + payload = payload.replace("$hawk-username", hawkUsername); + payload = payload.replace("$hawk-password", hawkPassword); + + payload = payload.replace("$gem-host", gemHost); + payload = payload.replace("$gem-port", Integer.toString(gemPort)); + payload = payload.replace("$gem-username", gemUsername); + payload = payload.replace("$gem-password", gemPassword); + payload = payload.replace("$gem-working-dir", gemWorkingDir); + + return payload; + } +} diff --git a/spring-cloud-pcf-connector/src/test/resources/log4j.properties b/spring-cloud-pcf-connector/src/test/resources/log4j.properties new file mode 100644 index 0000000..11492f1 --- /dev/null +++ b/spring-cloud-pcf-connector/src/test/resources/log4j.properties @@ -0,0 +1,6 @@ +log4j.rootCategory=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %40.40c:%4L - %m%n + diff --git a/spring-cloud-pcf-connector/src/test/resources/org/springframework/cloud/pcf/test-phd-info.json b/spring-cloud-pcf-connector/src/test/resources/org/springframework/cloud/pcf/test-phd-info.json new file mode 100644 index 0000000..a922f08 --- /dev/null +++ b/spring-cloud-pcf-connector/src/test/resources/org/springframework/cloud/pcf/test-phd-info.json @@ -0,0 +1,31 @@ +{ + "name":"$serviceName", + "label":"p-hd", + "tags":[], + "plan":"Standard", + "credentials":{ + "hadoop_username":"$hadoop-username", + "hdfs":{ + "configuration":{ + "fs.defaultFS":"hdfs://$hdfs-host:$hdfs-port" + }, + "directory":"$hdfs-directory" + }, + "yarn":{ + "configuration":{ + "yarn.resourcemanager.address":"$yarn-resource-host:$yarn-resource-port", + "mapreduce.framework.name":"yarn", + "yarn.resourcemanager.scheduler.address":"$yarn-resource-scheduler-host:$yarn-resource-scheduler-port", + "mapreduce.job.working.dir":"$yarn-map-reduce-dir", + "yarn.app.mapreduce.am.staging-dir":"$yarn-staging-dir" + } + }, + "hawq":{ + "uri":"jdbc:postgresql://$hawk-host:$hawk-port/postgres?user=$hawk-username&password=$hawk-password" + }, + "gemfirexd":{ + "uri":"jdbc:gemfirexd://$gem-host:$gem-port/;user=$gem-username;password=$gem-password", + "working.dir":"$gem-working-dir" + } + } +} \ No newline at end of file