BATCH-2003: Implemented:
* Basic partitioning support per JSR-352 * Parameter injection at the partition level * PartitionCollector/PartitionAnalyzer functionality * PartitionReducer functionality
This commit is contained in:
@@ -15,13 +15,15 @@
|
||||
*/
|
||||
package org.springframework.batch.core.jsr.configuration.support;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
@@ -35,6 +37,7 @@ public class BatchPropertyContextTests {
|
||||
private List<BatchPropertyContext.BatchPropertyContextEntry> stepProperties = new ArrayList<BatchPropertyContext.BatchPropertyContextEntry>();
|
||||
private List<BatchPropertyContext.BatchPropertyContextEntry> artifactProperties = new ArrayList<BatchPropertyContext.BatchPropertyContextEntry>();
|
||||
private List<BatchPropertyContext.BatchPropertyContextEntry> stepArtifactProperties = new ArrayList<BatchPropertyContext.BatchPropertyContextEntry>();
|
||||
private List<BatchPropertyContext.BatchPropertyContextEntry> partitionProperties = new ArrayList<BatchPropertyContext.BatchPropertyContextEntry>();
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
@@ -69,6 +72,26 @@ public class BatchPropertyContextTests {
|
||||
batchPropertyContextEntry.setStepName("step1");
|
||||
|
||||
this.stepArtifactProperties.add(batchPropertyContextEntry);
|
||||
|
||||
Properties partitionProperties = new Properties();
|
||||
partitionProperties.setProperty("writerProperty1", "writerProperty1valuePartition0");
|
||||
partitionProperties.setProperty("writerProperty2", "writerProperty2valuePartition0");
|
||||
|
||||
BatchPropertyContext.BatchPropertyContextEntry partitionBatchPropertyContextEntry =
|
||||
batchPropertyContext.new BatchPropertyContextEntry("writer", partitionProperties, BatchArtifact.BatchArtifactType.STEP_ARTIFACT);
|
||||
partitionBatchPropertyContextEntry.setStepName("step2:partition0");
|
||||
|
||||
this.partitionProperties.add(partitionBatchPropertyContextEntry);
|
||||
|
||||
Properties partitionStepProperties = new Properties();
|
||||
partitionStepProperties.setProperty("writerProperty1Step", "writerProperty1");
|
||||
partitionStepProperties.setProperty("writerProperty2Step", "writerProperty2");
|
||||
|
||||
BatchPropertyContext.BatchPropertyContextEntry partitionStepBatchPropertyContextEntry =
|
||||
batchPropertyContext.new BatchPropertyContextEntry("writer", partitionStepProperties, BatchArtifact.BatchArtifactType.STEP_ARTIFACT);
|
||||
partitionStepBatchPropertyContextEntry.setStepName("step2");
|
||||
|
||||
this.partitionProperties.add(partitionStepBatchPropertyContextEntry);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -179,4 +202,23 @@ public class BatchPropertyContextTests {
|
||||
assertEquals("jobProperty1value", job.getProperty("jobProperty1"));
|
||||
assertEquals("jobProperty2value", job.getProperty("jobProperty2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPartitionProperties() {
|
||||
BatchPropertyContext batchPropertyContext = new BatchPropertyContext();
|
||||
batchPropertyContext.setJobPropertiesContextEntry(jobProperties);
|
||||
batchPropertyContext.setArtifactPropertiesContextEntry(artifactProperties);
|
||||
batchPropertyContext.setStepPropertiesContextEntry(stepProperties);
|
||||
batchPropertyContext.setStepArtifactPropertiesContextEntry(stepArtifactProperties);
|
||||
batchPropertyContext.setStepArtifactPropertiesContextEntry(partitionProperties);
|
||||
|
||||
Properties artifactProperties = batchPropertyContext.getStepArtifactProperties("step2:partition0", "writer");
|
||||
assertEquals(6, artifactProperties.size());
|
||||
assertEquals("writerProperty1", artifactProperties.getProperty("writerProperty1Step"));
|
||||
assertEquals("writerProperty2", artifactProperties.getProperty("writerProperty2Step"));
|
||||
assertEquals("writerProperty1valuePartition0", artifactProperties.getProperty("writerProperty1"));
|
||||
assertEquals("writerProperty2valuePartition0", artifactProperties.getProperty("writerProperty2"));
|
||||
assertEquals("step2PropertyValue1", artifactProperties.getProperty("step2PropertyName1"));
|
||||
assertEquals("step2PropertyValue2", artifactProperties.getProperty("step2PropertyName2"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.batch.core.jsr.configuration.xml;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.Vector;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.batch.api.BatchProperty;
|
||||
import javax.batch.api.Batchlet;
|
||||
import javax.batch.api.chunk.AbstractItemReader;
|
||||
import javax.batch.api.chunk.AbstractItemWriter;
|
||||
import javax.batch.api.partition.PartitionPlan;
|
||||
import javax.batch.api.partition.PartitionPlanImpl;
|
||||
import javax.batch.operations.JobOperator;
|
||||
import javax.batch.runtime.BatchRuntime;
|
||||
import javax.batch.runtime.BatchStatus;
|
||||
import javax.batch.runtime.JobExecution;
|
||||
import javax.batch.runtime.context.JobContext;
|
||||
import javax.inject.Inject;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
public class PartitionParserTests {
|
||||
|
||||
private static JobOperator operator;
|
||||
private Pattern caPattern = Pattern.compile("ca");
|
||||
private Pattern asPattern = Pattern.compile("AS");
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() {
|
||||
operator = BatchRuntime.getJobOperator();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
MyBatchlet.processed = 0;
|
||||
MyBatchlet.threadNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
MyBatchlet.artifactNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
PartitionCollector.artifactNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBatchletNoProperties() {
|
||||
JobExecution execution = operator.getJobExecution(operator.start("partitionParserTestsBatchlet", new Properties()));
|
||||
|
||||
assertEquals(BatchStatus.COMPLETED, execution.getBatchStatus());
|
||||
assertEquals(10, MyBatchlet.processed);
|
||||
assertEquals(10, MyBatchlet.threadNames.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testChunkNoProperties() {
|
||||
JobExecution execution = operator.getJobExecution(operator.start("partitionParserTestsChunk", new Properties()));
|
||||
|
||||
assertEquals(BatchStatus.COMPLETED, execution.getBatchStatus());
|
||||
assertEquals(30, ItemReader.processedItems.size());
|
||||
assertEquals(10, ItemReader.threadNames.size());
|
||||
assertEquals(30, ItemWriter.processedItems.size());
|
||||
assertEquals(10, ItemWriter.threadNames.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullPartitionConfiguration() {
|
||||
JobExecution execution = operator.getJobExecution(operator.start("fullPartitionParserTests", new Properties()));
|
||||
|
||||
assertEquals(BatchStatus.COMPLETED, execution.getBatchStatus());
|
||||
assertTrue(execution.getExitStatus().startsWith("BPS_"));
|
||||
assertTrue(execution.getExitStatus().endsWith("BPSC_APSC"));
|
||||
assertEquals(3, countMatches(execution.getExitStatus(), caPattern));
|
||||
assertEquals(3, countMatches(execution.getExitStatus(), asPattern));
|
||||
assertEquals(3, MyBatchlet.processed);
|
||||
assertEquals(3, MyBatchlet.threadNames.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullPartitionConfigurationWithProperties() {
|
||||
JobExecution execution = operator.getJobExecution(operator.start("fullPartitionParserWithPropertiesTests", new Properties()));
|
||||
|
||||
assertEquals(BatchStatus.COMPLETED, execution.getBatchStatus());
|
||||
assertTrue(execution.getExitStatus().startsWith("BPS_"));
|
||||
assertTrue(execution.getExitStatus().endsWith("BPSC_APSC"));
|
||||
assertEquals(3, countMatches(execution.getExitStatus(), caPattern));
|
||||
assertEquals(3, countMatches(execution.getExitStatus(), asPattern));
|
||||
assertEquals(3, MyBatchlet.processed);
|
||||
assertEquals(3, MyBatchlet.threadNames.size());
|
||||
assertEquals(MyBatchlet.artifactNames.iterator().next(), "batchlet");
|
||||
assertEquals(PartitionMapper.name, "mapper");
|
||||
assertEquals(PartitionAnalyzer.name, "analyzer");
|
||||
assertEquals(PartitionReducer.name, "reducer");
|
||||
assertEquals(PartitionCollector.artifactNames.size(), 1);
|
||||
assertTrue(PartitionCollector.artifactNames.contains("collector"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullPartitionConfigurationWithMapperSuppliedProperties() {
|
||||
JobExecution execution = operator.getJobExecution(operator.start("fullPartitionParserWithMapperPropertiesTests", new Properties()));
|
||||
|
||||
assertEquals(BatchStatus.COMPLETED, execution.getBatchStatus());
|
||||
assertTrue(execution.getExitStatus().startsWith("BPS_"));
|
||||
assertTrue(execution.getExitStatus().endsWith("BPSC_APSC"));
|
||||
assertEquals(3, countMatches(execution.getExitStatus(), caPattern));
|
||||
assertEquals(3, countMatches(execution.getExitStatus(), asPattern));
|
||||
assertEquals(3, MyBatchlet.processed);
|
||||
assertEquals(3, MyBatchlet.threadNames.size());
|
||||
|
||||
assertEquals(MyBatchlet.artifactNames.size(), 3);
|
||||
assertTrue(MyBatchlet.artifactNames.contains("batchlet0"));
|
||||
assertTrue(MyBatchlet.artifactNames.contains("batchlet1"));
|
||||
assertTrue(MyBatchlet.artifactNames.contains("batchlet2"));
|
||||
assertEquals(PartitionCollector.artifactNames.size(), 3);
|
||||
assertTrue(PartitionCollector.artifactNames.contains("collector0"));
|
||||
assertTrue(PartitionCollector.artifactNames.contains("collector1"));
|
||||
assertTrue(PartitionCollector.artifactNames.contains("collector2"));
|
||||
|
||||
assertEquals(PartitionMapper.name, "mapper");
|
||||
assertEquals(PartitionAnalyzer.name, "analyzer");
|
||||
assertEquals(PartitionReducer.name, "reducer");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullPartitionConfigurationWithHardcodedProperties() {
|
||||
JobExecution execution = operator.getJobExecution(operator.start("fullPartitionParserWithHardcodedPropertiesTests", new Properties()));
|
||||
|
||||
assertEquals(BatchStatus.COMPLETED, execution.getBatchStatus());
|
||||
assertTrue(execution.getExitStatus().startsWith("BPS_"));
|
||||
assertTrue(execution.getExitStatus().endsWith("BPSC_APSC"));
|
||||
assertEquals(3, countMatches(execution.getExitStatus(), caPattern));
|
||||
assertEquals(3, countMatches(execution.getExitStatus(), asPattern));
|
||||
assertEquals(3, MyBatchlet.processed);
|
||||
assertEquals(3, MyBatchlet.threadNames.size());
|
||||
|
||||
assertEquals(MyBatchlet.artifactNames.size(), 3);
|
||||
assertTrue(MyBatchlet.artifactNames.contains("batchlet0"));
|
||||
assertTrue(MyBatchlet.artifactNames.contains("batchlet1"));
|
||||
assertTrue(MyBatchlet.artifactNames.contains("batchlet2"));
|
||||
assertEquals(PartitionCollector.artifactNames.size(), 3);
|
||||
assertTrue(PartitionCollector.artifactNames.contains("collector0"));
|
||||
assertTrue(PartitionCollector.artifactNames.contains("collector1"));
|
||||
assertTrue(PartitionCollector.artifactNames.contains("collector2"));
|
||||
|
||||
assertEquals(PartitionMapper.name, "mapper");
|
||||
assertEquals(PartitionAnalyzer.name, "analyzer");
|
||||
assertEquals(PartitionReducer.name, "reducer");
|
||||
}
|
||||
|
||||
private int countMatches(String string, Pattern pattern) {
|
||||
Matcher matcher = pattern.matcher(string);
|
||||
|
||||
int count = 0;
|
||||
while(matcher.find()) {
|
||||
count++;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
public static class PartitionReducer implements javax.batch.api.partition.PartitionReducer {
|
||||
|
||||
public static String name;
|
||||
|
||||
@Inject
|
||||
@BatchProperty
|
||||
String artifactName;
|
||||
|
||||
@Inject
|
||||
protected JobContext jobContext;
|
||||
|
||||
@Override
|
||||
public void beginPartitionedStep() throws Exception {
|
||||
name = artifactName;
|
||||
jobContext.setExitStatus("BPS_");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void beforePartitionedStepCompletion() throws Exception {
|
||||
jobContext.setExitStatus(jobContext.getExitStatus() + "BPSC_");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void rollbackPartitionedStep() throws Exception {
|
||||
jobContext.setExitStatus(jobContext.getExitStatus() + "RPS");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterPartitionedStepCompletion(PartitionStatus status)
|
||||
throws Exception {
|
||||
jobContext.setExitStatus(jobContext.getExitStatus() + "APSC");
|
||||
}
|
||||
}
|
||||
|
||||
public static class PartitionAnalyzer implements javax.batch.api.partition.PartitionAnalyzer {
|
||||
|
||||
public static String name;
|
||||
|
||||
@Inject
|
||||
@BatchProperty
|
||||
String artifactName;
|
||||
|
||||
@Inject
|
||||
protected JobContext jobContext;
|
||||
|
||||
@Override
|
||||
public void analyzeCollectorData(Serializable data) throws Exception {
|
||||
name = artifactName;
|
||||
|
||||
Assert.isTrue(data.equals("c"));
|
||||
jobContext.setExitStatus(jobContext.getExitStatus() + data + "a");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void analyzeStatus(BatchStatus batchStatus, String exitStatus)
|
||||
throws Exception {
|
||||
Assert.isTrue(batchStatus.equals(BatchStatus.COMPLETED));
|
||||
jobContext.setExitStatus(jobContext.getExitStatus() + "AS");
|
||||
}
|
||||
}
|
||||
|
||||
public static class PartitionCollector implements javax.batch.api.partition.PartitionCollector {
|
||||
|
||||
protected static Set<String> artifactNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
|
||||
@Inject
|
||||
@BatchProperty
|
||||
String artifactName;
|
||||
|
||||
@Override
|
||||
public Serializable collectPartitionData() throws Exception {
|
||||
artifactNames.add(artifactName);
|
||||
return "c";
|
||||
}
|
||||
}
|
||||
|
||||
public static class PropertyPartitionMapper implements javax.batch.api.partition.PartitionMapper {
|
||||
|
||||
@Override
|
||||
public PartitionPlan mapPartitions() throws Exception {
|
||||
Properties[] props = new Properties[3];
|
||||
|
||||
for(int i = 0; i < props.length; i++) {
|
||||
props[i] = new Properties();
|
||||
props[i].put("collectorName", "collector" + i);
|
||||
props[i].put("batchletName", "batchlet" + i);
|
||||
}
|
||||
|
||||
PartitionPlan plan = new PartitionPlanImpl();
|
||||
plan.setPartitions(3);
|
||||
plan.setThreads(3);
|
||||
plan.setPartitionProperties(props);
|
||||
|
||||
return plan;
|
||||
}
|
||||
}
|
||||
|
||||
public static class PartitionMapper implements javax.batch.api.partition.PartitionMapper {
|
||||
|
||||
public static String name;
|
||||
|
||||
@Inject
|
||||
@BatchProperty
|
||||
public String artifactName;
|
||||
|
||||
@Override
|
||||
public PartitionPlan mapPartitions() throws Exception {
|
||||
name = artifactName;
|
||||
|
||||
PartitionPlan plan = new PartitionPlanImpl();
|
||||
plan.setPartitions(3);
|
||||
plan.setThreads(3);
|
||||
|
||||
return plan;
|
||||
}
|
||||
}
|
||||
|
||||
public static class MyBatchlet implements Batchlet {
|
||||
|
||||
protected static int processed = 0;
|
||||
protected static Set<String> threadNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
protected static Set<String> artifactNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
|
||||
@Inject
|
||||
@BatchProperty
|
||||
String artifactName;
|
||||
|
||||
@Override
|
||||
public String process() throws Exception {
|
||||
artifactNames.add(artifactName);
|
||||
threadNames.add(Thread.currentThread().getName());
|
||||
processed++;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() throws Exception {
|
||||
}
|
||||
}
|
||||
|
||||
public static class ItemReader extends AbstractItemReader {
|
||||
|
||||
private List<Integer> items;
|
||||
protected static Vector<Integer> processedItems = new Vector<Integer>();
|
||||
protected static Set<String> threadNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
|
||||
@Override
|
||||
public void open(Serializable checkpoint) throws Exception {
|
||||
items = new ArrayList<Integer>();
|
||||
items.add(1);
|
||||
items.add(2);
|
||||
items.add(3);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object readItem() throws Exception {
|
||||
threadNames.add(Thread.currentThread().getName());
|
||||
if(items.size() > 0) {
|
||||
Integer curItem = items.remove(0);
|
||||
processedItems.add(curItem);
|
||||
return curItem;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class ItemWriter extends AbstractItemWriter {
|
||||
|
||||
protected static Vector<Integer> processedItems = new Vector<Integer>();
|
||||
protected static Set<String> threadNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
|
||||
@Override
|
||||
public void writeItems(List<Object> items) throws Exception {
|
||||
threadNames.add(Thread.currentThread().getName());
|
||||
for (Object object : items) {
|
||||
processedItems.add((Integer) object);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,18 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.batch.core.jsr.launch;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
@@ -341,6 +356,27 @@ public class JsrJobOperatorTests {
|
||||
jsrJobOperator.getStepExecutions(5l);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStepExecutionsPartitionedStepScenario() {
|
||||
JobExecution jobExecution = new JobExecution(5l);
|
||||
List<StepExecution> stepExecutions = new ArrayList<StepExecution>();
|
||||
stepExecutions.add(new StepExecution("step1", jobExecution, 1l));
|
||||
stepExecutions.add(new StepExecution("step2", jobExecution, 2l));
|
||||
stepExecutions.add(new StepExecution("step2:partition0", jobExecution, 2l));
|
||||
stepExecutions.add(new StepExecution("step2:partition1", jobExecution, 2l));
|
||||
stepExecutions.add(new StepExecution("step2:partition2", jobExecution, 2l));
|
||||
jobExecution.addStepExecutions(stepExecutions);
|
||||
|
||||
when(jobExplorer.getJobExecution(5l)).thenReturn(jobExecution);
|
||||
when(jobExplorer.getStepExecution(5l, 1l)).thenReturn(new StepExecution("step1", jobExecution, 1l));
|
||||
when(jobExplorer.getStepExecution(5l, 2l)).thenReturn(new StepExecution("step2", jobExecution, 2l));
|
||||
|
||||
List<javax.batch.runtime.StepExecution> results = jsrJobOperator.getStepExecutions(5l);
|
||||
|
||||
assertEquals("step1", results.get(0).getStepName());
|
||||
assertEquals("step2", results.get(1).getStepName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStepExecutionsNoStepExecutions() {
|
||||
JobExecution jobExecution = new JobExecution(5l);
|
||||
@@ -422,7 +458,7 @@ public class JsrJobOperatorTests {
|
||||
JsrJobOperator jobOperator = (JsrJobOperator) jsrJobOperator;
|
||||
|
||||
JobExecution jobExecution = new JobExecution(1L,
|
||||
new JobParametersBuilder().addString("prevKey1", "prevVal1").toJobParameters());
|
||||
new JobParametersBuilder().addString("prevKey1", "prevVal1").toJobParameters());
|
||||
|
||||
Properties userProperties = new Properties();
|
||||
userProperties.put("userKey1", "userVal1");
|
||||
@@ -440,9 +476,9 @@ public class JsrJobOperatorTests {
|
||||
|
||||
JobExecution jobExecution = new JobExecution(1L,
|
||||
new JobParametersBuilder()
|
||||
.addString("prevKey1", "prevVal1")
|
||||
.addString("overrideTest", "jobExecution")
|
||||
.toJobParameters());
|
||||
.addString("prevKey1", "prevVal1")
|
||||
.addString("overrideTest", "jobExecution")
|
||||
.toJobParameters());
|
||||
|
||||
Properties userProperties = new Properties();
|
||||
userProperties.put("userKey1", "userVal1");
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.batch.core.jsr.partition;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Collection;
|
||||
import java.util.Properties;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
|
||||
import javax.batch.api.partition.PartitionAnalyzer;
|
||||
import javax.batch.api.partition.PartitionMapper;
|
||||
import javax.batch.api.partition.PartitionPlan;
|
||||
import javax.batch.api.partition.PartitionPlanImpl;
|
||||
import javax.batch.runtime.BatchStatus;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.springframework.batch.core.ExitStatus;
|
||||
import org.springframework.batch.core.JobExecution;
|
||||
import org.springframework.batch.core.JobInterruptedException;
|
||||
import org.springframework.batch.core.StepExecution;
|
||||
import org.springframework.batch.core.jsr.configuration.support.BatchPropertyContext;
|
||||
import org.springframework.batch.core.partition.JsrStepExecutionSplitter;
|
||||
import org.springframework.batch.core.repository.JobRepository;
|
||||
import org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean;
|
||||
import org.springframework.batch.core.step.JobRepositorySupport;
|
||||
import org.springframework.batch.core.step.StepSupport;
|
||||
|
||||
public class JsrPartitionHandlerTests {
|
||||
|
||||
private JsrPartitionHandler handler;
|
||||
private JobRepository repository = new JobRepositorySupport();
|
||||
private StepExecution stepExecution = new StepExecution("step", new JobExecution(1L));
|
||||
private int count;
|
||||
private BatchPropertyContext propertyContext;
|
||||
private JsrStepExecutionSplitter stepSplitter;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
stepSplitter = new JsrStepExecutionSplitter("step1", repository);
|
||||
Analyzer.collectorData = "";
|
||||
Analyzer.status = "";
|
||||
count = 0;
|
||||
handler = new JsrPartitionHandler();
|
||||
handler.setStep(new StepSupport() {
|
||||
@Override
|
||||
public void execute(StepExecution stepExecution) throws JobInterruptedException {
|
||||
count++;
|
||||
stepExecution.setStatus(org.springframework.batch.core.BatchStatus.COMPLETED);
|
||||
stepExecution.setExitStatus(new ExitStatus("done"));
|
||||
}
|
||||
});
|
||||
propertyContext = new BatchPropertyContext();
|
||||
handler.setPropertyContext(propertyContext);
|
||||
repository = new MapJobRepositoryFactoryBean().getJobRepository();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAfterPropertiesSet() throws Exception {
|
||||
handler = new JsrPartitionHandler();
|
||||
|
||||
try {
|
||||
handler.afterPropertiesSet();
|
||||
fail("PropertyContext was not checked for");
|
||||
} catch(IllegalArgumentException iae) {
|
||||
assertEquals("A BatchPropertyContext is required", iae.getMessage());
|
||||
}
|
||||
|
||||
handler.setPropertyContext(new BatchPropertyContext());
|
||||
|
||||
try {
|
||||
handler.afterPropertiesSet();
|
||||
fail("Threads or mapper was not checked for");
|
||||
} catch(IllegalArgumentException iae) {
|
||||
assertEquals("Either a mapper implementation or the number of partitions/threads is required", iae.getMessage());
|
||||
}
|
||||
|
||||
handler.setThreads(3);
|
||||
handler.afterPropertiesSet();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHardcodedNumberOfPartitions() throws Exception {
|
||||
handler.setThreads(3);
|
||||
handler.setPartitions(3);
|
||||
handler.afterPropertiesSet();
|
||||
|
||||
Collection<StepExecution> executions = handler.handle(stepSplitter, stepExecution);
|
||||
|
||||
assertEquals(3, executions.size());
|
||||
assertEquals(3, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapperProvidesPartitions() throws Exception {
|
||||
handler.setPartitionMapper(new PartitionMapper() {
|
||||
|
||||
@Override
|
||||
public PartitionPlan mapPartitions() throws Exception {
|
||||
PartitionPlan plan = new PartitionPlanImpl();
|
||||
plan.setPartitions(3);
|
||||
plan.setThreads(0);
|
||||
return plan;
|
||||
}
|
||||
});
|
||||
|
||||
handler.afterPropertiesSet();
|
||||
|
||||
Collection<StepExecution> executions = handler.handle(new JsrStepExecutionSplitter("step1", repository), stepExecution);
|
||||
|
||||
assertEquals(3, executions.size());
|
||||
assertEquals(3, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapperProvidesPartitionsAndThreads() throws Exception {
|
||||
handler.setPartitionMapper(new PartitionMapper() {
|
||||
|
||||
@Override
|
||||
public PartitionPlan mapPartitions() throws Exception {
|
||||
PartitionPlan plan = new PartitionPlanImpl();
|
||||
plan.setPartitions(3);
|
||||
plan.setThreads(1);
|
||||
return plan;
|
||||
}
|
||||
});
|
||||
|
||||
handler.afterPropertiesSet();
|
||||
|
||||
Collection<StepExecution> executions = handler.handle(new JsrStepExecutionSplitter("step1", repository), stepExecution);
|
||||
|
||||
assertEquals(3, executions.size());
|
||||
assertEquals(3, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapperWithProperties() throws Exception {
|
||||
handler.setPartitionMapper(new PartitionMapper() {
|
||||
|
||||
@Override
|
||||
public PartitionPlan mapPartitions() throws Exception {
|
||||
PartitionPlan plan = new PartitionPlanImpl();
|
||||
Properties [] props = new Properties[2];
|
||||
props[0] = new Properties();
|
||||
props[0].put("key1", "value1");
|
||||
props[1] = new Properties();
|
||||
props[1].put("key1", "value2");
|
||||
plan.setPartitionProperties(props);
|
||||
plan.setPartitions(3);
|
||||
plan.setThreads(1);
|
||||
return plan;
|
||||
}
|
||||
});
|
||||
|
||||
handler.afterPropertiesSet();
|
||||
|
||||
Collection<StepExecution> executions = handler.handle(new JsrStepExecutionSplitter("step1", repository), stepExecution);
|
||||
|
||||
assertEquals(3, executions.size());
|
||||
assertEquals(3, count);
|
||||
assertEquals("value1", propertyContext.getStepProperties("step1:partition0").get("key1"));
|
||||
assertEquals("value2", propertyContext.getStepProperties("step1:partition1").get("key1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAnalyzer() throws Exception {
|
||||
Queue<Serializable> queue = new ConcurrentLinkedQueue<Serializable>();
|
||||
queue.add("foo");
|
||||
queue.add("bar");
|
||||
|
||||
handler.setPartitionDataQueue(queue);
|
||||
handler.setThreads(2);
|
||||
handler.setPartitions(2);
|
||||
handler.setPartitionAnalyzer(new Analyzer());
|
||||
handler.afterPropertiesSet();
|
||||
|
||||
Collection<StepExecution> executions = handler.handle(new JsrStepExecutionSplitter("step1", repository), stepExecution);
|
||||
|
||||
assertEquals(2, executions.size());
|
||||
assertEquals(2, count);
|
||||
assertEquals("foobar", Analyzer.collectorData);
|
||||
assertEquals("COMPLETEDdone", Analyzer.status);
|
||||
}
|
||||
|
||||
public static class Analyzer implements PartitionAnalyzer {
|
||||
|
||||
public static String collectorData;
|
||||
public static String status;
|
||||
|
||||
@Override
|
||||
public void analyzeCollectorData(Serializable data) throws Exception {
|
||||
collectorData = collectorData + data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void analyzeStatus(BatchStatus batchStatus, String exitStatus)
|
||||
throws Exception {
|
||||
status = batchStatus + exitStatus;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.batch.core.jsr.partition;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.springframework.batch.core.JobExecution;
|
||||
import org.springframework.batch.core.StepExecution;
|
||||
import org.springframework.batch.core.partition.JsrStepExecutionSplitter;
|
||||
import org.springframework.batch.core.step.JobRepositorySupport;
|
||||
|
||||
public class JsrStepExecutionSplitterTests {
|
||||
|
||||
private JsrStepExecutionSplitter splitter;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
splitter = new JsrStepExecutionSplitter("step1", new JobRepositorySupport());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
Set<StepExecution> executions = splitter.split(new StepExecution("step1", new JobExecution(5l)), 3);
|
||||
|
||||
assertEquals(3, executions.size());
|
||||
|
||||
Iterator<StepExecution> stepExecutions = executions.iterator();
|
||||
|
||||
int count = 0;
|
||||
while(stepExecutions.hasNext()) {
|
||||
StepExecution curExecution = stepExecutions.next();
|
||||
assertEquals("step1:partition" + count, curExecution.getStepName());
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.batch.core.jsr.partition;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
|
||||
import javax.batch.api.partition.PartitionCollector;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class PartitionCollectorAdapterTests {
|
||||
|
||||
private PartitionCollectorAdapter adapter;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
adapter = new PartitionCollectorAdapter();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPropertiesSeet() throws Exception {
|
||||
try {
|
||||
adapter.afterPropertiesSet();
|
||||
fail("Did not check for a PartitionCollector instance");
|
||||
} catch (IllegalArgumentException iae) {
|
||||
assertEquals(iae.getMessage(), "A PartitionCollector instance is required");
|
||||
}
|
||||
|
||||
adapter.setPartitionCollector(new PartitionCollector() {
|
||||
|
||||
@Override
|
||||
public Serializable collectPartitionData() throws Exception {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
adapter.afterPropertiesSet();
|
||||
fail("Did not check for a queue");
|
||||
} catch (IllegalArgumentException iae) {
|
||||
assertEquals(iae.getMessage(), "A thread safe Queue instance is required");
|
||||
}
|
||||
|
||||
adapter.setPartitionQueue(new ConcurrentLinkedQueue<Serializable>());
|
||||
|
||||
adapter.afterPropertiesSet();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAfterChunkSuccessful() throws Exception {
|
||||
adapter.setPartitionCollector(new PartitionCollector() {
|
||||
|
||||
private int count = 0;
|
||||
|
||||
@Override
|
||||
public Serializable collectPartitionData() throws Exception {
|
||||
return String.valueOf(count++);
|
||||
}
|
||||
});
|
||||
|
||||
Queue<Serializable> dataQueue = new ConcurrentLinkedQueue<Serializable>();
|
||||
adapter.setPartitionQueue(dataQueue);
|
||||
adapter.afterPropertiesSet();
|
||||
|
||||
adapter.afterChunk(null);
|
||||
adapter.afterChunk(null);
|
||||
adapter.afterChunk(null);
|
||||
|
||||
assertEquals(3, dataQueue.size());
|
||||
assertEquals("0", dataQueue.remove());
|
||||
assertEquals("1", dataQueue.remove());
|
||||
assertEquals("2", dataQueue.remove());
|
||||
}
|
||||
|
||||
@Test(expected=PartitionException.class)
|
||||
public void testAfterChunkException() {
|
||||
// Throws an NPE due to a null collector
|
||||
adapter.afterChunk(null);
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import org.springframework.batch.core.JobInterruptedException;
|
||||
import org.springframework.batch.core.JobParameters;
|
||||
import org.springframework.batch.core.Step;
|
||||
import org.springframework.batch.core.StepExecution;
|
||||
import org.springframework.batch.core.jsr.configuration.support.BatchPropertyContext;
|
||||
import org.springframework.batch.core.jsr.step.builder.JsrSimpleStepBuilder;
|
||||
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
|
||||
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
|
||||
@@ -68,6 +69,7 @@ public class JsrChunkProcessorTests {
|
||||
readListener = new CountingListener();
|
||||
|
||||
builder = new JsrSimpleStepBuilder<String, String>(new StepBuilder("step1"));
|
||||
builder.setBatchPropertyContext(new BatchPropertyContext());
|
||||
repository = new MapJobRepositoryFactoryBean().getJobRepository();
|
||||
builder.repository(repository);
|
||||
builder.transactionManager(new ResourcelessTransactionManager());
|
||||
|
||||
@@ -32,6 +32,7 @@ import org.springframework.batch.core.JobInterruptedException;
|
||||
import org.springframework.batch.core.JobParameters;
|
||||
import org.springframework.batch.core.Step;
|
||||
import org.springframework.batch.core.StepExecution;
|
||||
import org.springframework.batch.core.jsr.configuration.support.BatchPropertyContext;
|
||||
import org.springframework.batch.core.jsr.step.builder.JsrFaultTolerantStepBuilder;
|
||||
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
|
||||
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
|
||||
@@ -69,6 +70,7 @@ public class JsrFaultTolerantChunkProcessorTests {
|
||||
readListener = new CountingListener();
|
||||
|
||||
builder = new JsrFaultTolerantStepBuilder<String, String>(new StepBuilder("step1"));
|
||||
builder.setBatchPropertyContext(new BatchPropertyContext());
|
||||
repository = new MapJobRepositoryFactoryBean().getJobRepository();
|
||||
builder.repository(repository);
|
||||
builder.transactionManager(new ResourcelessTransactionManager());
|
||||
|
||||
@@ -23,6 +23,8 @@ import static org.junit.Assert.fail;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.springframework.batch.core.JobExecution;
|
||||
@@ -30,6 +32,9 @@ import org.springframework.batch.core.JobInstance;
|
||||
import org.springframework.batch.core.JobParameters;
|
||||
import org.springframework.batch.core.JobParametersBuilder;
|
||||
import org.springframework.batch.core.StepExecution;
|
||||
import org.springframework.batch.core.jsr.configuration.support.BatchArtifact.BatchArtifactType;
|
||||
import org.springframework.batch.core.jsr.configuration.support.BatchPropertyContext;
|
||||
import org.springframework.batch.core.jsr.configuration.support.BatchPropertyContext.BatchPropertyContextEntry;
|
||||
import org.springframework.batch.item.ExecutionContext;
|
||||
|
||||
/**
|
||||
@@ -44,6 +49,8 @@ public class StepContextTests {
|
||||
|
||||
private StepContext context = new StepContext(stepExecution);
|
||||
|
||||
private BatchPropertyContext propertyContext = new BatchPropertyContext();
|
||||
|
||||
@Test
|
||||
public void testGetStepExecution() {
|
||||
context = new StepContext(stepExecution);
|
||||
@@ -61,6 +68,22 @@ public class StepContextTests {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetPartitionPlan() {
|
||||
Properties partitionPropertyValues = new Properties();
|
||||
partitionPropertyValues.put("key1", "value1");
|
||||
List<BatchPropertyContextEntry> entries = new ArrayList<BatchPropertyContext.BatchPropertyContextEntry>();
|
||||
BatchPropertyContextEntry entry = propertyContext.new BatchPropertyContextEntry(stepExecution.getStepName(), partitionPropertyValues, BatchArtifactType.STEP);
|
||||
entries.add(entry);
|
||||
|
||||
propertyContext.setStepPropertiesContextEntry(entries);
|
||||
|
||||
context = new StepContext(stepExecution, propertyContext);
|
||||
|
||||
Map<String, Object> plan = context.getPartitionPlan();
|
||||
assertEquals("value1", plan.get("key1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEqualsSelf() {
|
||||
assertEquals(context, context);
|
||||
|
||||
@@ -1,3 +1,18 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.batch.core.scope.context;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
@@ -19,11 +34,13 @@ import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.springframework.batch.core.JobExecution;
|
||||
import org.springframework.batch.core.StepExecution;
|
||||
import org.springframework.batch.core.jsr.configuration.support.BatchPropertyContext;
|
||||
import org.springframework.util.ReflectionUtils;
|
||||
|
||||
public class StepSynchronizationManagerTests {
|
||||
|
||||
private StepExecution stepExecution = new StepExecution("step", new JobExecution(0L));
|
||||
private BatchPropertyContext propertyContext = new BatchPropertyContext();
|
||||
|
||||
@Before
|
||||
@After
|
||||
@@ -40,6 +57,16 @@ public class StepSynchronizationManagerTests {
|
||||
assertNotNull(StepSynchronizationManager.getContext());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetContextWithBatchProperties() {
|
||||
StepContext context = StepSynchronizationManager.getContext();
|
||||
assertNull(context);
|
||||
StepSynchronizationManager.register(stepExecution, propertyContext);
|
||||
context = StepSynchronizationManager.getContext();
|
||||
assertNotNull(context);
|
||||
assertEquals(stepExecution, context.getStepExecution());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClose() throws Exception {
|
||||
final List<String> list = new ArrayList<String>();
|
||||
|
||||
Reference in New Issue
Block a user