From 025c447b89cc2dddc27cdb36e3dcc45a238611f3 Mon Sep 17 00:00:00 2001 From: Michael Minella Date: Mon, 22 Aug 2016 11:46:29 -0400 Subject: [PATCH] Refactored for CloudFoundry Support This commit adds on an abstraction for the providing unique command line arugements for each worker. It also updates the partitioned job sample to be able to successfully be run on CloudFoundry. Resolves spring-cloud/spring-cloud-task#193 Updated based on comments --- .../partition/CommandLineArgsProvider.java | 42 ++++++++++++ .../partition/DeployerPartitionHandler.java | 68 +++++++++++++++++-- .../NoOpEnvironmentVariablesProvider.java | 4 +- .../PassThroughCommandLineArgsProvider.java | 43 ++++++++++++ .../SimpleCommandLineArgsProvider.java | 44 ++++++++++++ .../SimpleEnvironmentVariablesProvider.java | 20 +++++- .../DeployerPartitionHandlerTests.java | 32 +++++++-- ...NoOpEnvironmentVariablesProviderTests.java | 45 ++++++++++++ ...ssThroughCommandLineArgsProviderTests.java | 46 +++++++++++++ .../SimpleCommandLineArgsProviderTests.java | 50 ++++++++++++++ .../task/launcher/TaskLauncherSinkTests.java | 8 +-- .../partitioned-batch-job/README.adoc | 2 +- .../partitioned-batch-job/pom.xml | 4 +- .../main/java/io/spring/JobConfiguration.java | 37 +++++----- .../src/main/resources/application.properties | 3 +- 15 files changed, 402 insertions(+), 46 deletions(-) create mode 100644 spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/CommandLineArgsProvider.java create mode 100644 spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/PassThroughCommandLineArgsProvider.java create mode 100644 spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/SimpleCommandLineArgsProvider.java create mode 100644 spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/NoOpEnvironmentVariablesProviderTests.java create mode 100644 spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/PassThroughCommandLineArgsProviderTests.java create mode 100644 spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/SimpleCommandLineArgsProviderTests.java diff --git a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/CommandLineArgsProvider.java b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/CommandLineArgsProvider.java new file mode 100644 index 00000000..37635142 --- /dev/null +++ b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/CommandLineArgsProvider.java @@ -0,0 +1,42 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.task.batch.partition; + +import java.util.List; + +import org.springframework.batch.item.ExecutionContext; + +/** + * Strategy to allow for the customization of command line arguments passed to each + * partition's execution. + * + * @author Michael Minella + * @since 1.1.0 + */ +public interface CommandLineArgsProvider { + + /** + * Returns a unique list of command line arguements to be passed to the partition's + * worker for the specified {@link ExecutionContext}. + * + * Note: This method is called once per partition. + * + * @param executionContext the unique state for the step to be executed. + * @return a list of formatted command line arguments to be passed to the worker (the + * list will be joined via spaces). + */ + List getCommandLineArgs(ExecutionContext executionContext); +} diff --git a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/DeployerPartitionHandler.java b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/DeployerPartitionHandler.java index 9e764966..85d65956 100644 --- a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/DeployerPartitionHandler.java +++ b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/DeployerPartitionHandler.java @@ -47,6 +47,7 @@ import org.springframework.core.env.Environment; import org.springframework.core.io.Resource; import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; /** *

A {@link PartitionHandler} implementation that delegates to a {@link TaskLauncher} for @@ -75,6 +76,8 @@ public class DeployerPartitionHandler implements PartitionHandler, EnvironmentAw public static final String SPRING_CLOUD_TASK_STEP_NAME = "spring.cloud.task.step-name"; + public static final String SPRING_CLOUD_TASK_NAME = "spring.cloud.task.name"; + private int maxWorkers = -1; private int gridSize = 1; @@ -103,6 +106,10 @@ public class DeployerPartitionHandler implements PartitionHandler, EnvironmentAw private EnvironmentVariablesProvider environmentVariablesProvider; + private String applicationName; + + private CommandLineArgsProvider commandLineArgsProvider; + public DeployerPartitionHandler(TaskLauncher taskLauncher, JobExplorer jobExplorer, Resource resource, @@ -118,10 +125,24 @@ public class DeployerPartitionHandler implements PartitionHandler, EnvironmentAw this.stepName = stepName; } + /** + * Used to provide any environment variables to be set on each worker launched. + * + * @param environmentVariablesProvider an {@link EnvironmentVariablesProvider} + */ public void setEnvironmentVariablesProvider(EnvironmentVariablesProvider environmentVariablesProvider) { this.environmentVariablesProvider = environmentVariablesProvider; } + /** + * Used to provide any command line arguements to be passed to each worker launched. + * + * @param commandLineArgsProvider {@link CommandLineArgsProvider} + */ + public void setCommandLineArgsProvider(CommandLineArgsProvider commandLineArgsProvider) { + this.commandLineArgsProvider = commandLineArgsProvider; + } + /** * The maximum number of workers to be executing at once. * @@ -170,9 +191,24 @@ public class DeployerPartitionHandler implements PartitionHandler, EnvironmentAw this.deploymentProperties = deploymentProperties; } + /** + * The name of the application to be launched. Useful in environments where + * application deployments are reused (such as CloudFoundry). + * + * @param applicationName The name of the application to be launched + */ + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + @BeforeTask public void beforeTask(TaskExecution taskExecution) { this.taskExecution = taskExecution; + + if(this.commandLineArgsProvider == null) { + this.commandLineArgsProvider = + new SimpleCommandLineArgsProvider(this.taskExecution); + } } @Override @@ -216,21 +252,30 @@ public class DeployerPartitionHandler implements PartitionHandler, EnvironmentAw private void launchWorker(StepExecution workerStepExecution) { List arguments = new ArrayList<>(); - arguments.addAll(this.taskExecution.getArguments()); + + ExecutionContext copyContext = new ExecutionContext(workerStepExecution.getExecutionContext()); + + arguments.addAll( + this.commandLineArgsProvider + .getCommandLineArgs(copyContext)); + arguments.add(formatArgument(SPRING_CLOUD_TASK_JOB_EXECUTION_ID, String.valueOf(workerStepExecution.getJobExecution().getId()))); arguments.add(formatArgument(SPRING_CLOUD_TASK_STEP_EXECUTION_ID, String.valueOf(workerStepExecution.getId()))); arguments.add(formatArgument(SPRING_CLOUD_TASK_STEP_NAME, this.stepName)); + arguments.add(formatArgument(SPRING_CLOUD_TASK_NAME, String.format("%s:%s:%s", + taskExecution.getTaskName(), + workerStepExecution.getJobExecution().getJobInstance().getJobName(), + workerStepExecution.getStepName()))); - ExecutionContext copyContext = new ExecutionContext(workerStepExecution.getExecutionContext()); + copyContext = new ExecutionContext(workerStepExecution.getExecutionContext()); + + Map environmentVariables = this.environmentVariablesProvider.getEnvironmentVariables(copyContext); AppDefinition definition = - new AppDefinition(String.format("%s:%s:%s", - taskExecution.getTaskName(), - workerStepExecution.getJobExecution().getJobInstance().getJobName(), - workerStepExecution.getStepName()), - this.environmentVariablesProvider.getEnvironmentVariables(copyContext)); + new AppDefinition(resolveApplicationName(), + environmentVariables); AppDeploymentRequest request = new AppDeploymentRequest(definition, @@ -241,6 +286,15 @@ public class DeployerPartitionHandler implements PartitionHandler, EnvironmentAw taskLauncher.launch(request); } + private String resolveApplicationName() { + if(StringUtils.hasText(this.applicationName)) { + return this.applicationName; + } + else { + return this.taskExecution.getTaskName(); + } + } + private String formatArgument(String key, String value) { return String.format("--%s=%s", key, value); } diff --git a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/NoOpEnvironmentVariablesProvider.java b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/NoOpEnvironmentVariablesProvider.java index 1e42d0e7..02b49f63 100644 --- a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/NoOpEnvironmentVariablesProvider.java +++ b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/NoOpEnvironmentVariablesProvider.java @@ -15,7 +15,7 @@ */ package org.springframework.cloud.task.batch.partition; -import java.util.HashMap; +import java.util.Collections; import java.util.Map; import org.springframework.batch.item.ExecutionContext; @@ -38,6 +38,6 @@ public class NoOpEnvironmentVariablesProvider implements EnvironmentVariablesPro */ @Override public Map getEnvironmentVariables(ExecutionContext executionContext) { - return new HashMap<>(0); + return Collections.emptyMap(); } } diff --git a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/PassThroughCommandLineArgsProvider.java b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/PassThroughCommandLineArgsProvider.java new file mode 100644 index 00000000..e2ef58d4 --- /dev/null +++ b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/PassThroughCommandLineArgsProvider.java @@ -0,0 +1,43 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.task.batch.partition; + +import java.util.List; + +import org.springframework.batch.item.ExecutionContext; +import org.springframework.util.Assert; + +/** + * Returns the {@code List} provided. + * + * @author Michael Minella + * @since 1.1.0 + */ +public class PassThroughCommandLineArgsProvider implements CommandLineArgsProvider { + + private final List commandLineArgs; + + public PassThroughCommandLineArgsProvider(List commandLineArgs) { + Assert.notNull(commandLineArgs, "commandLineArgs is required"); + + this.commandLineArgs = commandLineArgs; + } + + @Override + public List getCommandLineArgs(ExecutionContext executionContext) { + return commandLineArgs; + } +} diff --git a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/SimpleCommandLineArgsProvider.java b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/SimpleCommandLineArgsProvider.java new file mode 100644 index 00000000..15538d25 --- /dev/null +++ b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/SimpleCommandLineArgsProvider.java @@ -0,0 +1,44 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.task.batch.partition; + +import java.util.List; + +import org.springframework.batch.item.ExecutionContext; +import org.springframework.cloud.task.repository.TaskExecution; +import org.springframework.util.Assert; + +/** + * Returns any command line arguments used with the {@link TaskExecution} provided. + * + * @author Michael Minella + * @since 1.1.0 + */ +public class SimpleCommandLineArgsProvider implements CommandLineArgsProvider { + + private final TaskExecution taskExecution; + + public SimpleCommandLineArgsProvider(TaskExecution taskExecution) { + Assert.notNull(taskExecution, "A taskExecution is required"); + + this.taskExecution = taskExecution; + } + + @Override + public List getCommandLineArgs(ExecutionContext executionContext) { + return this.taskExecution.getArguments(); + } +} diff --git a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/SimpleEnvironmentVariablesProvider.java b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/SimpleEnvironmentVariablesProvider.java index 0451c334..c75b33a4 100644 --- a/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/SimpleEnvironmentVariablesProvider.java +++ b/spring-cloud-task-batch/src/main/java/org/springframework/cloud/task/batch/partition/SimpleEnvironmentVariablesProvider.java @@ -28,7 +28,8 @@ import org.springframework.core.env.MapPropertySource; import org.springframework.core.env.PropertySource; /** - * Copies all existing environment variables as made available in the {@link Environment}. + * Copies all existing environment variables as made available in the {@link Environment} + * only if includeCurrentEnvironment is set to true (default). * The environmentProperties option provides the ability to override any * specific values on an as needed basis. * @@ -42,6 +43,8 @@ public class SimpleEnvironmentVariablesProvider implements EnvironmentVariablesP private Map environmentProperties = new HashMap<>(0); + private boolean includeCurrentEnvironment = true; + /** * @param environment The {@link Environment} for this context */ @@ -57,11 +60,24 @@ public class SimpleEnvironmentVariablesProvider implements EnvironmentVariablesP this.environmentProperties = environmentProperties; } + /** + * Establishes if current environment variables will be included as a part of the provider. + * @param includeCurrentEnvironment true(default) include local environment properties. False do not include + * current environment properties. + */ + public void setIncludeCurrentEnvironment(boolean includeCurrentEnvironment) { + this.includeCurrentEnvironment = includeCurrentEnvironment; + } + @Override public Map getEnvironmentVariables(ExecutionContext executionContext) { Map environmentProperties = new HashMap<>(this.environmentProperties.size()); - environmentProperties.putAll(getCurrentEnvironmentProperties()); + + if(includeCurrentEnvironment) { + environmentProperties.putAll(getCurrentEnvironmentProperties()); + } + environmentProperties.putAll(this.environmentProperties); return environmentProperties; diff --git a/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/DeployerPartitionHandlerTests.java b/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/DeployerPartitionHandlerTests.java index 417a4d1d..acfc1c21 100644 --- a/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/DeployerPartitionHandlerTests.java +++ b/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/DeployerPartitionHandlerTests.java @@ -149,7 +149,7 @@ public class DeployerPartitionHandlerTests { AppDefinition appDefinition = request.getDefinition(); - assertEquals("partitionedJobTask:partitionedJob:step1:partition1", appDefinition.getName()); + assertEquals("partitionedJobTask", appDefinition.getName()); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_JOB_EXECUTION_ID, "1"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_EXECUTION_ID, "4"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_NAME, "step1"))); @@ -362,7 +362,7 @@ public class DeployerPartitionHandlerTests { AppDefinition appDefinition = request.getDefinition(); - assertEquals("partitionedJobTask:partitionedJob:step1:partition1", appDefinition.getName()); + assertEquals("partitionedJobTask", appDefinition.getName()); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_JOB_EXECUTION_ID, "1"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_EXECUTION_ID, "4"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_NAME, "step1"))); @@ -426,7 +426,7 @@ public class DeployerPartitionHandlerTests { AppDefinition appDefinition = request.getDefinition(); - assertEquals("partitionedJobTask:partitionedJob:step1:partition1", appDefinition.getName()); + assertEquals("partitionedJobTask", appDefinition.getName()); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_JOB_EXECUTION_ID, "1"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_EXECUTION_ID, "4"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_NAME, "step1"))); @@ -608,7 +608,7 @@ public class DeployerPartitionHandlerTests { AppDefinition appDefinition = request.getDefinition(); - assertEquals("partitionedJobTask:partitionedJob:step1:partition1", appDefinition.getName()); + assertEquals("partitionedJobTask", appDefinition.getName()); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_JOB_EXECUTION_ID, "1"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_EXECUTION_ID, "4"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_NAME, "step1"))); @@ -661,7 +661,27 @@ public class DeployerPartitionHandlerTests { Collections.sort(allRequests, new Comparator() { @Override public int compare(AppDeploymentRequest o1, AppDeploymentRequest o2) { - return o1.getDefinition().getName().compareTo(o2.getDefinition().getName()); + List commandlineArguments = o1.getCommandlineArguments(); + + String o1Command = ""; + for (String commandlineArgument : commandlineArguments) { + if(commandlineArgument.contains(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_EXECUTION_ID)) { + o1Command = commandlineArgument; + break; + } + } + + commandlineArguments = o2.getCommandlineArguments(); + + String o2Command = ""; + for (String commandlineArgument : commandlineArguments) { + if(commandlineArgument.contains(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_EXECUTION_ID)) { + o2Command = commandlineArgument; + break; + } + } + + return o1Command.compareTo(o2Command); } }); @@ -671,7 +691,7 @@ public class DeployerPartitionHandlerTests { assertEquals(0, request.getDeploymentProperties().size()); AppDefinition appDefinition = request.getDefinition(); - assertEquals("partitionedJobTask:partitionedJob:step1:partition" + (i - 3), appDefinition.getName()); + assertEquals("partitionedJobTask", appDefinition.getName()); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_JOB_EXECUTION_ID, "1"))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_EXECUTION_ID, String.valueOf(i)))); assertTrue(request.getCommandlineArguments().contains(formatArgs(DeployerPartitionHandler.SPRING_CLOUD_TASK_STEP_NAME, "step1"))); diff --git a/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/NoOpEnvironmentVariablesProviderTests.java b/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/NoOpEnvironmentVariablesProviderTests.java new file mode 100644 index 00000000..65e695db --- /dev/null +++ b/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/NoOpEnvironmentVariablesProviderTests.java @@ -0,0 +1,45 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.task.batch.partition; + +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * @author Michael Minella + */ +public class NoOpEnvironmentVariablesProviderTests { + + private NoOpEnvironmentVariablesProvider provider; + + @Before + public void setUp() { + this.provider = new NoOpEnvironmentVariablesProvider(); + } + + @Test + public void test() { + Map environmentVariables = this.provider.getEnvironmentVariables(null); + assertNotNull(environmentVariables); + assertTrue(environmentVariables.isEmpty()); + + Map environmentVariables2 = this.provider.getEnvironmentVariables(null); + assertTrue(environmentVariables == environmentVariables2); + } +} diff --git a/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/PassThroughCommandLineArgsProviderTests.java b/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/PassThroughCommandLineArgsProviderTests.java new file mode 100644 index 00000000..aeb38b9f --- /dev/null +++ b/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/PassThroughCommandLineArgsProviderTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.task.batch.partition; + +import java.util.Arrays; +import java.util.List; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * @author Michael Minella + */ +public class PassThroughCommandLineArgsProviderTests { + + @Test(expected = IllegalArgumentException.class) + public void testNull() { + new PassThroughCommandLineArgsProvider(null); + } + + @Test + public void test() { + List args = Arrays.asList("foo", "bar", "baz"); + + CommandLineArgsProvider provider = new PassThroughCommandLineArgsProvider(args); + + List commandLineArgs = provider.getCommandLineArgs(null); + + assertEquals("foo", commandLineArgs.get(0)); + assertEquals("bar", commandLineArgs.get(1)); + assertEquals("baz", commandLineArgs.get(2)); + } +} diff --git a/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/SimpleCommandLineArgsProviderTests.java b/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/SimpleCommandLineArgsProviderTests.java new file mode 100644 index 00000000..8146aef4 --- /dev/null +++ b/spring-cloud-task-batch/src/test/java/org/springframework/cloud/task/batch/partition/SimpleCommandLineArgsProviderTests.java @@ -0,0 +1,50 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.task.batch.partition; + +import java.util.Arrays; +import java.util.List; + +import org.junit.Test; + +import org.springframework.cloud.task.repository.TaskExecution; + +import static org.junit.Assert.assertEquals; + +/** + * @author Michael Minella + */ +public class SimpleCommandLineArgsProviderTests { + + @Test(expected = IllegalArgumentException.class) + public void testNullConstructorArg() { + new SimpleCommandLineArgsProvider(null); + } + + @Test + public void test() { + TaskExecution taskExecution = new TaskExecution(); + taskExecution.setArguments(Arrays.asList("foo", "bar", "baz")); + + CommandLineArgsProvider provider = new SimpleCommandLineArgsProvider(taskExecution); + + List commandLineArgs = provider.getCommandLineArgs(null); + + assertEquals("foo", commandLineArgs.get(0)); + assertEquals("bar", commandLineArgs.get(1)); + assertEquals("baz", commandLineArgs.get(2)); + } +} diff --git a/spring-cloud-task-integration-tests/src/test/java/org/springframework/cloud/task/launcher/TaskLauncherSinkTests.java b/spring-cloud-task-integration-tests/src/test/java/org/springframework/cloud/task/launcher/TaskLauncherSinkTests.java index 4913a54b..0413085e 100644 --- a/spring-cloud-task-integration-tests/src/test/java/org/springframework/cloud/task/launcher/TaskLauncherSinkTests.java +++ b/spring-cloud-task-integration-tests/src/test/java/org/springframework/cloud/task/launcher/TaskLauncherSinkTests.java @@ -30,7 +30,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.SpringApplicationConfiguration; +import org.springframework.boot.test.context.SpringBootTest; import org.springframework.cloud.deployer.spi.local.LocalDeployerProperties; import org.springframework.cloud.deployer.spi.local.LocalTaskLauncher; import org.springframework.cloud.deployer.spi.task.TaskLauncher; @@ -52,14 +52,14 @@ import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.jdbc.datasource.init.DataSourceInitializer; import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.junit4.SpringRunner; import org.springframework.util.SocketUtils; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -@RunWith(SpringJUnit4ClassRunner.class) -@SpringApplicationConfiguration(classes = {TaskLauncherSinkApplication.class, TaskLauncherSinkTests.TaskLauncherConfiguration.class}) +@RunWith(SpringRunner.class) +@SpringBootTest(classes = {TaskLauncherSinkApplication.class, TaskLauncherSinkTests.TaskLauncherConfiguration.class}) public class TaskLauncherSinkTests { private final static int WAIT_INTERVAL = 500; diff --git a/spring-cloud-task-samples/partitioned-batch-job/README.adoc b/spring-cloud-task-samples/partitioned-batch-job/README.adoc index 76cf0a3a..312291f1 100644 --- a/spring-cloud-task-samples/partitioned-batch-job/README.adoc +++ b/spring-cloud-task-samples/partitioned-batch-job/README.adoc @@ -22,7 +22,7 @@ $ export spring_datasource_url=jdbc:mysql://localhost:3306/ $ export spring_datasource_username= $ export spring_datasource_password= $ export spring_datasource_driverClassName=org.mariadb.jdbc.Driver -$ java -jar -Dspring.profiles.active=master target/partitioned-batch-job-1.1.0.BUILD-SNAPSHOT.jar +$ java -jar target/partitioned-batch-job-1.1.0.BUILD-SNAPSHOT.jar ---- NOTE: This example will use require a MySql RDBMS repository and currently uses the mariadb jdbc driver to connect. diff --git a/spring-cloud-task-samples/partitioned-batch-job/pom.xml b/spring-cloud-task-samples/partitioned-batch-job/pom.xml index 1463e029..3545ee2e 100644 --- a/spring-cloud-task-samples/partitioned-batch-job/pom.xml +++ b/spring-cloud-task-samples/partitioned-batch-job/pom.xml @@ -64,8 +64,8 @@ com.h2database h2 + test - org.springframework.boot spring-boot-starter-jdbc @@ -77,7 +77,7 @@ test - + diff --git a/spring-cloud-task-samples/partitioned-batch-job/src/main/java/io/spring/JobConfiguration.java b/spring-cloud-task-samples/partitioned-batch-job/src/main/java/io/spring/JobConfiguration.java index cc99955b..ca133dab 100644 --- a/spring-cloud-task-samples/partitioned-batch-job/src/main/java/io/spring/JobConfiguration.java +++ b/spring-cloud-task-samples/partitioned-batch-job/src/main/java/io/spring/JobConfiguration.java @@ -15,8 +15,11 @@ */ package io.spring; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Random; import javax.sql.DataSource; import org.springframework.batch.core.Job; @@ -37,11 +40,11 @@ import org.springframework.batch.repeat.RepeatStatus; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.deployer.resource.support.DelegatingResourceLoader; -import org.springframework.cloud.deployer.spi.local.LocalDeployerProperties; -import org.springframework.cloud.deployer.spi.local.LocalTaskLauncher; import org.springframework.cloud.deployer.spi.task.TaskLauncher; import org.springframework.cloud.task.batch.partition.DeployerPartitionHandler; import org.springframework.cloud.task.batch.partition.DeployerStepExecutionHandler; +import org.springframework.cloud.task.batch.partition.NoOpEnvironmentVariablesProvider; +import org.springframework.cloud.task.batch.partition.PassThroughCommandLineArgsProvider; import org.springframework.cloud.task.batch.partition.SimpleEnvironmentVariablesProvider; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; @@ -88,29 +91,20 @@ public class JobConfiguration { return jobExplorerFactoryBean; } - @Bean - public TaskLauncher taskLauncher() { - LocalDeployerProperties localDeployerProperties = new LocalDeployerProperties(); - - localDeployerProperties.setDeleteFilesOnExit(false); - - return new LocalTaskLauncher(localDeployerProperties); - } - @Bean public PartitionHandler partitionHandler(TaskLauncher taskLauncher, JobExplorer jobExplorer) throws Exception { Resource resource = resourceLoader.getResource("maven://io.spring.cloud:partitioned-batch-job:1.1.0.BUILD-SNAPSHOT"); DeployerPartitionHandler partitionHandler = new DeployerPartitionHandler(taskLauncher, jobExplorer, resource, "workerStep"); - Map environmentProperties = new HashMap<>(); - environmentProperties.put("spring.profiles.active", "worker"); - - SimpleEnvironmentVariablesProvider environmentVariablesProvider = new SimpleEnvironmentVariablesProvider(this.environment); - environmentVariablesProvider.setEnvironmentProperties(environmentProperties); - partitionHandler.setEnvironmentVariablesProvider(environmentVariablesProvider); - - partitionHandler.setMaxWorkers(2); + List commandLineArgs = new ArrayList<>(3); + commandLineArgs.add("--spring.profiles.active=worker"); + commandLineArgs.add("--spring.cloud.task.initialize.enable=false"); + commandLineArgs.add("--spring.batch.initializer.enabled=false"); + partitionHandler.setCommandLineArgsProvider(new PassThroughCommandLineArgsProvider(commandLineArgs)); + partitionHandler.setEnvironmentVariablesProvider(new SimpleEnvironmentVariablesProvider(this.environment)); + partitionHandler.setMaxWorkers(1); + partitionHandler.setApplicationName("PartitionedBatchJobTask"); return partitionHandler; } @@ -173,9 +167,10 @@ public class JobConfiguration { } @Bean - @Profile("master") + @Profile("!worker") public Job partitionedJob(PartitionHandler partitionHandler) throws Exception { - return jobBuilderFactory.get("partitionedJob") + Random random = new Random(); + return jobBuilderFactory.get("partitionedJob"+random.nextInt()) .start(step1(partitionHandler)) .build(); } diff --git a/spring-cloud-task-samples/partitioned-batch-job/src/main/resources/application.properties b/spring-cloud-task-samples/partitioned-batch-job/src/main/resources/application.properties index db26a862..711f1bae 100644 --- a/spring-cloud-task-samples/partitioned-batch-job/src/main/resources/application.properties +++ b/spring-cloud-task-samples/partitioned-batch-job/src/main/resources/application.properties @@ -1,3 +1,4 @@ -spring.application.name=Partitioned Batch Job Task +spring.application.name=PartitionedBatchJobTask logging.level.org.springframework.cloud.task=DEBUG maven.remoteRepositories.springRepo.url=https://repo.spring.io/libs-snapshot +