SpringCloud/spring-cloud-connectors
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support PHD service on pcf

Browse Source 
- Hadoop along with yarn
- Hawq
- Gemfirexd
This commit is contained in:
Ramnivas Laddad
2014-08-22 13:11:29 -07:00
parent 3ca9c8a43e
commit ca0ae812f1
15 changed files with 452 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
settings.gradle
View File

@@ -5,3 +5,4 @@ include "${rootProject.name}-cloudfoundry-connector"
include "${rootProject.name}-spring-service-connector"
include "${rootProject.name}-heroku-connector"
include "${rootProject.name}-localconfig-connector"
include "${rootProject.name}-pcf-connector"

2
spring-cloud-pcf-connector/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
dependency-reduced-pom.xml
/bin

3
spring-cloud-pcf-connector/README.md Normal file
View File

@@ -0,0 +1,3 @@
#Spring Cloud PCF extension
Currently supports the Hadoop service on Pivotal Cloud Foundry

8
spring-cloud-pcf-connector/build.gradle Normal file
View File

@@ -0,0 +1,8 @@
description = 'Spring-Cloud Support for Pivotal CF'
dependencies {
compile project(':spring-cloud-cloudfoundry-connector')
optional("org.apache.hadoop:hadoop-common:2.2.0")
testCompile project(path: ':spring-cloud-cloudfoundry-connector', configuration: 'tests')
}

65
spring-cloud-pcf-connector/publish-maven.gradle Normal file
View File

@@ -0,0 +1,65 @@
apply plugin: "maven"
ext.optionalDeps = []
ext.providedDeps = []
ext.optional = { optionalDeps << it }
ext.provided = { providedDeps << it }
install {
repositories.mavenInstaller {
customizePom(pom, project)
}
}
def customizePom(pom, gradleProject) {
pom.whenConfigured { generatedPom ->
// respect "optional" and "provided" dependencies
gradleProject.optionalDeps.each { dep ->
generatedPom.dependencies.findAll { it.artifactId == dep.name }*.optional = true
}
gradleProject.providedDeps.each { dep ->
generatedPom.dependencies.findAll { it.artifactId == dep.name }*.scope = "provided"
}
// eliminate test-scoped dependencies (no need in maven central poms)
generatedPom.dependencies.removeAll { dep ->
dep.scope == "test"
}
// Remove jackson dependencies, since we shade them in
generatedPom.dependencies.removeAll { dep ->
dep.groupId == "com.fasterxml.jackson.core"
}
// add all items necessary for maven central publication
generatedPom.project {
name = gradleProject.description
description = gradleProject.description
url = "https://github.com/spring-projects/spring-cloud"
organization {
name = "Spring IO"
url = "http://projects.spring.io/spring-cloud"
}
licenses {
license {
name "The Apache Software License, Version 2.0"
url "http://www.apache.org/licenses/LICENSE-2.0.txt"
distribution "repo"
}
}
scm {
url = "https://github.com/spring-projects/spring-cloud"
connection = "scm:git:git://github.com/spring-projects/spring-cloud"
developerConnection = "scm:git:git://github.com/spring-projects/spring-cloud"
}
developers {
developer {
id = "ramnivas"
name = "Ramnivas Laddad"
email = ""
}
}
}
}
}

20
spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/gemfire/GemfireXDServiceInfo.java Normal file
View File

@@ -0,0 +1,20 @@
package org.springframework.cloud.pcf.gemfire;
import org.springframework.cloud.service.ServiceInfo.ServiceLabel;
import org.springframework.cloud.service.common.RelationalServiceInfo;
/**
*
* @author Ramnivas Laddad
*
*/
@ServiceLabel("gemfirexd")
public class GemfireXDServiceInfo extends RelationalServiceInfo {
public static final String JDBC_URL_TYPE = "gemfirexd";
public GemfireXDServiceInfo(String id, String url) {
super(id, url, JDBC_URL_TYPE);
}
}

25
spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/hadoop/HadoopConfigurationCreator.java Normal file
View File

@@ -0,0 +1,25 @@
package org.springframework.cloud.pcf.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.springframework.cloud.service.AbstractServiceConnectorCreator;
import org.springframework.cloud.service.ServiceConnectorConfig;
/**
*
* @author Ramnivas Laddad
*
*/
public class HadoopConfigurationCreator extends AbstractServiceConnectorCreator<Configuration, HadoopServiceInfo> {
@Override
public Configuration create(HadoopServiceInfo hadoopServiceInfo, ServiceConnectorConfig serviceConnectorConfig) {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", hadoopServiceInfo.getDefaultFS());
configuration.set("yarn.resourcemanager.address", hadoopServiceInfo.getYarnResourceManagerAddress());
configuration.set("yarn.resourcemanager.scheduler.address", hadoopServiceInfo.getYarnResourceManagerSchedulerAddress());
return configuration;
}
}

42
spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/hadoop/HadoopServiceInfo.java Normal file
View File

@@ -0,0 +1,42 @@
package org.springframework.cloud.pcf.hadoop;
import org.springframework.cloud.service.BaseServiceInfo;
import org.springframework.cloud.service.ServiceInfo;
/**
* Hadoop service info (also considers yarn properties).
*
* @author Ramnivas Laddad
*
*/
public class HadoopServiceInfo extends BaseServiceInfo {
private String defaultHdfsUri;
private String yarnResourceManagerAddress;
private String yarnResourceManagerSchedulerAddress;
public HadoopServiceInfo(String id, String defaultHdfsUri,
String yarnResourceManagerAddress, String yarnResourceManagerSchedulerAddress) {
super(id);
this.defaultHdfsUri = defaultHdfsUri;
this.yarnResourceManagerAddress = yarnResourceManagerAddress;
this.yarnResourceManagerSchedulerAddress = yarnResourceManagerSchedulerAddress;
}
@ServiceInfo.ServiceProperty(category = "connection")
public String getDefaultFS() {
return defaultHdfsUri;
}
@ServiceInfo.ServiceProperty(category = "connection")
public String getYarnResourceManagerAddress() {
return yarnResourceManagerAddress;
}
@ServiceInfo.ServiceProperty(category = "connection")
public String getYarnResourceManagerSchedulerAddress() {
return yarnResourceManagerSchedulerAddress;
}
}

79
spring-cloud-pcf-connector/src/main/java/org/springframework/cloud/pcf/phd/PhdServiceInfoCreator.java Normal file
View File

@@ -0,0 +1,79 @@
package org.springframework.cloud.pcf.phd;
import java.util.Map;
import org.springframework.cloud.cloudfoundry.CloudFoundryServiceInfoCreator;
import org.springframework.cloud.cloudfoundry.Tags;
import org.springframework.cloud.pcf.gemfire.GemfireXDServiceInfo;
import org.springframework.cloud.pcf.hadoop.HadoopServiceInfo;
import org.springframework.cloud.service.BaseCompositeServiceInfo;
import org.springframework.cloud.service.ServiceInfo;
import org.springframework.cloud.service.common.PostgresqlServiceInfo;
import org.springframework.cloud.service.common.RelationalServiceInfo;
/**
*
* @author Ramnivas Laddad
*
*/
public class PhdServiceInfoCreator extends CloudFoundryServiceInfoCreator<BaseCompositeServiceInfo> {
public PhdServiceInfoCreator() {
super(new Tags("p-hd"));
}
@SuppressWarnings("unchecked")
public BaseCompositeServiceInfo createServiceInfo(Map<String,Object> serviceData) {
String id = (String) serviceData.get("name");
Map<String,Object> credentials = (Map<String, Object>) serviceData.get("credentials");
ServiceInfo hadoopServiceInfo = createHadoopServiceInfo(id, credentials);
ServiceInfo hawqServiceInfo = createHawqServiceInfo(id, credentials);
ServiceInfo gemfirexdServiceInfo = createGemfireXDServiceInfo(id, credentials);
return new BaseCompositeServiceInfo(id, hadoopServiceInfo, hawqServiceInfo, gemfirexdServiceInfo);
}
@SuppressWarnings("unchecked")
private HadoopServiceInfo createHadoopServiceInfo(String id, Map<String,Object> credentials) {
Map<String, Object> hdfs = (Map<String, Object>) credentials.get("hdfs");
Map<String, Object> hdfsConfig = (Map<String, Object>) hdfs.get("configuration");
String defaultHdfsUri = (String) hdfsConfig.get("fs.defaultFS");
Map<String, Object> yarn = (Map<String, Object>) credentials.get("yarn");
Map<String, Object> yarnConfig = (Map<String, Object>) yarn.get("configuration");
String yarnResourceManagerAddress = (String) yarnConfig.get("yarn.resourcemanager.address");
String yarnResourceManagerSchedulerAddress = (String) yarnConfig.get("yarn.resourcemanager.scheduler.address");
return new HadoopServiceInfo(id + "/hadoop", defaultHdfsUri,
yarnResourceManagerAddress, yarnResourceManagerSchedulerAddress);
}
private RelationalServiceInfo createHawqServiceInfo(String id, Map<String,Object> credentials) {
String key = "hawq";
return new PostgresqlServiceInfo(id + "/" + key, extractDataSourceUri(id, key, credentials));
}
private RelationalServiceInfo createGemfireXDServiceInfo(String id, Map<String,Object> credentials) {
String key = "gemfirexd";
return new GemfireXDServiceInfo(id + "/" + key, extractDataSourceUri(id, key, credentials));
}
private String extractDataSourceUri(String id, String key, Map<String,Object> credentials) {
@SuppressWarnings("unchecked")
Map<String, Object> hawq = (Map<String, Object>) credentials.get(key);
String uri = (String) hawq.get("uri");
String jdbcUriPrefix = "jdbc:";
if (uri.startsWith(jdbcUriPrefix)) {
uri.substring(jdbcUriPrefix.length());
}
return uri;
}
}

1
spring-cloud-pcf-connector/src/main/resources/META-INF/services/org.springframework.cloud.cloudfoundry.CloudFoundryServiceInfoCreator Normal file
View File

@@ -0,0 +1 @@
org.springframework.cloud.pcf.phd.PhdServiceInfoCreator

1
spring-cloud-pcf-connector/src/main/resources/META-INF/services/org.springframework.cloud.service.ServiceConnectorCreator Normal file
View File

@@ -0,0 +1 @@
org.springframework.cloud.pcf.hadoop.HadoopConfigurationCreator

26
spring-cloud-pcf-connector/src/test/java/org/springframework/cloud/hadoop/HadoopConfigurationCreatorTest.java Normal file
View File

@@ -0,0 +1,26 @@
package org.springframework.cloud.hadoop;
import static org.junit.Assert.*;
import org.apache.hadoop.conf.Configuration;
import org.junit.Test;
import org.springframework.cloud.pcf.hadoop.HadoopConfigurationCreator;
import org.springframework.cloud.pcf.hadoop.HadoopServiceInfo;
/**
*
* @author Ramnivas Laddad
*
*/
public class HadoopConfigurationCreatorTest {
private HadoopConfigurationCreator creator = new HadoopConfigurationCreator();
@Test
public void hadoopConfigurationCreation() {
HadoopServiceInfo serviceInfo = new HadoopServiceInfo("phd-hadoop", "hdfs://hdfshost:1234", "yrm:2345", "yrsm:3456");
Configuration configuration = creator.create(serviceInfo, null);
assertEquals("hdfs://hdfshost:1234", configuration.get("fs.defaultFS"));
assertEquals("yrm:2345", configuration.get("yarn.resourcemanager.address"));
assertEquals("yrsm:3456", configuration.get("yarn.resourcemanager.scheduler.address"));
}
}

142
spring-cloud-pcf-connector/src/test/java/org/springframework/cloud/pcf/PhdServiceInfoCreatorTest.java Normal file
View File

@@ -0,0 +1,142 @@
package org.springframework.cloud.pcf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.when;
import java.util.List;
import org.junit.Test;
import org.springframework.cloud.cloudfoundry.AbstractCloudFoundryConnectorTest;
import org.springframework.cloud.pcf.gemfire.GemfireXDServiceInfo;
import org.springframework.cloud.pcf.hadoop.HadoopServiceInfo;
import org.springframework.cloud.service.BaseCompositeServiceInfo;
import org.springframework.cloud.service.ServiceInfo;
import org.springframework.cloud.service.common.PostgresqlServiceInfo;
/**
*
* @author Ramnivas Laddad
*
*/
public class PhdServiceInfoCreatorTest extends AbstractCloudFoundryConnectorTest {
@Test
public void phdServiceCreation() {
String hadoopUsername = "hadoop-user";
String hadoopHost = "hadoop-host";
int hadoopPort = 6000;
String hadoopDirectory = "hadoop-dir";
String yarnResourceHost = "yark-resource-host";
int yarnResourcePort = 7000;
String yarnResourceSchedulerHost = "yarn-resource-scheduler-host";
int yarnResourceSchedulerPort = 8000;
String yarnMapReduceDir = "yarn-map-reduce-dir";
String yarnStagingDir = "yarn-staging-dir";
String hawkHost = "hawk-host";
int hawkPort = 9000;
String hawkUsername = "hawk-user";
String hawkPassword = "hawk-pass";
String gemHost = "gem-host";
int gemPort = 10000;
String gemUsername = "gem-user";
String gemPassword = "gem-pass";
String gemWorkingDir = "gem-working-dir";
when(mockEnvironment.getEnvValue("VCAP_SERVICES")).thenReturn(
getServicesPayload(
getPhdServicePayload("phd-1", hadoopUsername, hadoopHost, hadoopPort, hadoopDirectory,
yarnResourceHost, yarnResourcePort,
yarnResourceSchedulerHost, yarnResourceSchedulerPort, yarnMapReduceDir, yarnStagingDir,
hawkHost, hawkPort, hawkUsername, hawkPassword,
gemHost, gemPort, gemUsername, gemPassword, gemWorkingDir)));
List<ServiceInfo> serviceInfos = testCloudConnector.getServiceInfos();
BaseCompositeServiceInfo phdServiceInfo = (BaseCompositeServiceInfo) getServiceInfo(serviceInfos, "phd-1");
assertNotNull(phdServiceInfo);
assertEquals(3, phdServiceInfo.getServiceInfos().size());
HadoopServiceInfo hadoopServiceInfo = extractServiceInfo(phdServiceInfo.getServiceInfos(), HadoopServiceInfo.class);
assertNotNull(hadoopServiceInfo);
assertHadoopServiceInfo("phd-1/hadoop", hadoopHost, hadoopPort,
yarnResourceHost, yarnResourcePort, yarnResourceSchedulerHost, yarnResourceSchedulerPort, hadoopServiceInfo);
PostgresqlServiceInfo hawqServiceInfo = extractServiceInfo(phdServiceInfo.getServiceInfos(), PostgresqlServiceInfo.class);
assertEquals("phd-1/hawq", hawqServiceInfo.getId());
assertNotNull(hawqServiceInfo);
GemfireXDServiceInfo gemfireXDServiceInfo = extractServiceInfo(phdServiceInfo.getServiceInfos(), GemfireXDServiceInfo.class);
assertNotNull(gemfireXDServiceInfo);
assertEquals("phd-1/gemfirexd", gemfireXDServiceInfo.getId());
}
private void assertHadoopServiceInfo(String serviceId, String hadoopHost, int hadoopPort,
String yarnResourceHost, int yarnResourcePort,
String yarnResourceSchedulerHost, int yarnResourceSchedulerPort,
HadoopServiceInfo serviceInfo) {
assertEquals(serviceId, serviceInfo.getId());
assertEquals(String.format("hdfs://%s:%s", hadoopHost, hadoopPort), serviceInfo.getDefaultFS());
assertEquals(String.format("%s:%s", yarnResourceHost, yarnResourcePort), serviceInfo.getYarnResourceManagerAddress());
assertEquals(String.format("%s:%s", yarnResourceSchedulerHost, yarnResourceSchedulerPort), serviceInfo.getYarnResourceManagerSchedulerAddress());
}
@SuppressWarnings("unchecked")
private <T> T extractServiceInfo(List<ServiceInfo> serviceInfos, Class<T> typeToSearch) {
for (ServiceInfo serviceInfo: serviceInfos) {
if (serviceInfo.getClass().equals(typeToSearch)) {
return (T)serviceInfo;
}
}
return null;
}
private String getPhdServicePayload(String serviceName,
String hadoopUsername, String hadoopHost, int hadoopPort, String hadoopDirectory,
String yarnResourceHost, int yarnResourcePort,
String yarnResourceSchedulerHost, int yarnResourceSchedulerPort,
String yarnMapReduceDir, String yarnStagingDir,
String hawkHost, int hawkPort, String hawkUsername, String hawkPassword,
String gemHost, int gemPort, String gemUsername, String gemPassword, String gemWorkingDir) {
return getPhdServicePayload("test-phd-info.json", serviceName, hadoopUsername, hadoopHost, hadoopPort, hadoopDirectory,
yarnResourceHost, yarnResourcePort,
yarnResourceSchedulerHost, yarnResourceSchedulerPort,
yarnMapReduceDir, yarnStagingDir,
hawkHost, hawkPort, hawkUsername, hawkPassword,
gemHost, gemPort, gemUsername, gemPassword, gemWorkingDir);
}
private String getPhdServicePayload(String filename, String serviceName,
String hadoopUsername, String hadoopHost, int hadoopPort, String hadoopDirectory,
String yarnResourceHost, int yarnResourcePort,
String yarnResourceSchedulerHost, int yarnResourceSchedulerPort,
String yarnMapReduceDir, String yarnStagingDir,
String hawkHost, int hawkPort, String hawkUsername, String hawkPassword,
String gemHost, int gemPort, String gemUsername, String gemPassword, String gemWorkingDir) {
String payload = readTestDataFile(filename);
payload = payload.replace("$serviceName", serviceName);
payload = payload.replace("$hadoop-username", hadoopUsername);
payload = payload.replace("$hdfs-host", hadoopHost);
payload = payload.replace("$hdfs-port", Integer.toString(hadoopPort));
payload = payload.replace("$hdfs-directory", hadoopDirectory);
payload = payload.replace("$yarn-resource-host", yarnResourceHost);
payload = payload.replace("$yarn-resource-port", Integer.toString(yarnResourcePort));
payload = payload.replace("$yarn-resource-scheduler-host", yarnResourceSchedulerHost);
payload = payload.replace("$yarn-resource-scheduler-port", Integer.toString(yarnResourceSchedulerPort));
payload = payload.replace("$yarn-map-reduce-dir", yarnMapReduceDir);
payload = payload.replace("$yarn-staging-dir", yarnStagingDir);
payload = payload.replace("$hawk-host", hawkHost);
payload = payload.replace("$hawk-port", Integer.toString(hawkPort));
payload = payload.replace("$hawk-username", hawkUsername);
payload = payload.replace("$hawk-password", hawkPassword);
payload = payload.replace("$gem-host", gemHost);
payload = payload.replace("$gem-port", Integer.toString(gemPort));
payload = payload.replace("$gem-username", gemUsername);
payload = payload.replace("$gem-password", gemPassword);
payload = payload.replace("$gem-working-dir", gemWorkingDir);
return payload;
}
}

6
spring-cloud-pcf-connector/src/test/resources/log4j.properties Normal file
View File

@@ -0,0 +1,6 @@
log4j.rootCategory=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %40.40c:%4L - %m%n

31
spring-cloud-pcf-connector/src/test/resources/org/springframework/cloud/pcf/test-phd-info.json Normal file
View File

@@ -0,0 +1,31 @@
{
"name":"$serviceName",
"label":"p-hd",
"tags":[],
"plan":"Standard",
"credentials":{
"hadoop_username":"$hadoop-username",
"hdfs":{
"configuration":{
"fs.defaultFS":"hdfs://$hdfs-host:$hdfs-port"
},
"directory":"$hdfs-directory"
},
"yarn":{
"configuration":{
"yarn.resourcemanager.address":"$yarn-resource-host:$yarn-resource-port",
"mapreduce.framework.name":"yarn",
"yarn.resourcemanager.scheduler.address":"$yarn-resource-scheduler-host:$yarn-resource-scheduler-port",
"mapreduce.job.working.dir":"$yarn-map-reduce-dir",
"yarn.app.mapreduce.am.staging-dir":"$yarn-staging-dir"
}
},
"hawq":{
"uri":"jdbc:postgresql://$hawk-host:$hawk-port/postgres?user=$hawk-username&password=$hawk-password"
},
"gemfirexd":{
"uri":"jdbc:gemfirexd://$gem-host:$gem-port/;user=$gem-username;password=$gem-password",
"working.dir":"$gem-working-dir"
}
}
}