Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request 134 from jpraet/BATCH-1957

Browse Source 
* BATCH-1957:
  BATCH-1957: Updated comments
  - restore WRITE_STATISTICS_NAME from execution context on restart - don't assume output file is always present on restart (could be deleted by previous execution if shouldDeleteIfEmpty flag was set)
  add additional test case that shows a bug in the current StaxEventItemWriter.shouldDeleteIfEmpty: when the writer restarts from a previous execution where the output file was deleted, this fails because FileUtils.setUpOutputFile expects the file to be there on restart
  add unit test test showing bug in FlatFileItemWriter: output file is deleted on restart if shouldDeleteIfEmpty property is set and no new records have been written
  BATCH-1957: Add StaxEventItemWriter deleteIfEmpty property
This commit is contained in:
Michael Minella
2013-03-20 10:37:12 -05:00
parent b83d88df98 00f1d31127
commit 29c6968865
4 changed files with 235 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/FlatFileItemWriter.java
View File

@@ -469,7 +469,14 @@ InitializingBean {
*/
public void restoreFrom(ExecutionContext executionContext) {
lastMarkedByteOffsetPosition = executionContext.getLong(getExecutionContextKey(RESTART_DATA_NAME));
restarted = true;
linesWritten = executionContext.getLong(getExecutionContextKey(WRITTEN_STATISTICS_NAME));
if (shouldDeleteIfEmpty && linesWritten == 0) {
// previous execution deleted the output file because no items were written
restarted = false;
lastMarkedByteOffsetPosition = 0;
} else {
restarted = true;
}
}
/**
@@ -585,7 +592,6 @@ InitializingBean {
}
initialized = true;
linesWritten = 0;
}
public boolean isInitialized() {

28
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/xml/StaxEventItemWriter.java
View File

@@ -142,6 +142,8 @@ ResourceAwareItemWriterItemStream<T>, InitializingBean {
private boolean forceSync;
private boolean shouldDeleteIfEmpty = false;
public StaxEventItemWriter() {
setExecutionContextName(ClassUtils.getShortName(StaxEventItemWriter.class));
}
@@ -203,6 +205,16 @@ ResourceAwareItemWriterItemStream<T>, InitializingBean {
this.forceSync = forceSync;
}
/**
* Flag to indicate that the target file should be deleted if no items have
* been written (other than header and footer) on close. Defaults to false.
*
* @param shouldDeleteIfEmpty the flag value to set
*/
public void setShouldDeleteIfEmpty(boolean shouldDeleteIfEmpty) {
this.shouldDeleteIfEmpty = shouldDeleteIfEmpty;
}
/**
* Get used encoding.
*
@@ -354,7 +366,15 @@ ResourceAwareItemWriterItemStream<T>, InitializingBean {
// otherwise start from beginning
if (executionContext.containsKey(getExecutionContextKey(RESTART_DATA_NAME))) {
startAtPosition = executionContext.getLong(getExecutionContextKey(RESTART_DATA_NAME));
currentRecordCount = executionContext.getLong(getExecutionContextKey(WRITE_STATISTICS_NAME));
restarted = true;
if (shouldDeleteIfEmpty && currentRecordCount == 0) {
// previous execution deleted the output file because no items were written
restarted = false;
startAtPosition = 0;
} else {
restarted = true;
}
}
open(startAtPosition, restarted);
@@ -656,6 +676,14 @@ ResourceAwareItemWriterItemStream<T>, InitializingBean {
}
}
}
if (currentRecordCount == 0 && shouldDeleteIfEmpty) {
try {
resource.getFile().delete();
}
catch (IOException e) {
throw new ItemStreamException("Failed to delete empty file on close", e);
}
}
}
}

46
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemWriterTests.java
View File

@@ -323,8 +323,8 @@ public class FlatFileItemWriterTests {
assertEquals("footer", readLine());
// 3 lines were written to the file after restart
assertEquals(3, executionContext.getLong(ClassUtils.getShortName(FlatFileItemWriter.class) + ".written"));
// 8 lines were written to the file in total
assertEquals(8, executionContext.getLong(ClassUtils.getShortName(FlatFileItemWriter.class) + ".written"));
}
@@ -425,8 +425,8 @@ public class FlatFileItemWriterTests {
assertEquals("footer", readLine());
// 3 lines were written to the file after restart
assertEquals(3, executionContext.getLong(ClassUtils.getShortName(FlatFileItemWriter.class) + ".written"));
// 8 lines were written to the file in total
assertEquals(8, executionContext.getLong(ClassUtils.getShortName(FlatFileItemWriter.class) + ".written"));
}
@@ -595,6 +595,7 @@ public class FlatFileItemWriterTests {
public void testDeleteOnExitReopen() throws Exception {
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.update(executionContext);
assertTrue(outputFile.exists());
writer.close();
assertFalse(outputFile.exists());
@@ -602,6 +603,43 @@ public class FlatFileItemWriterTests {
writer.write(Collections.singletonList("test2"));
assertEquals("test2", readLine());
}
@Test
public void testWriteHeaderAndDeleteOnExitReopen() throws Exception {
writer.setHeaderCallback(new FlatFileHeaderCallback() {
@Override
public void writeHeader(Writer writer) throws IOException {
writer.write("a\nb");
}
});
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.update(executionContext);
assertTrue(outputFile.exists());
writer.close();
assertFalse(outputFile.exists());
writer.open(executionContext);
writer.write(Collections.singletonList("test2"));
assertEquals("a", readLine());
assertEquals("b", readLine());
assertEquals("test2", readLine());
}
@Test
public void testDeleteOnExitNoRecordsWrittenAfterRestart() throws Exception {
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.write(Collections.singletonList("test2"));
writer.update(executionContext);
writer.close();
assertTrue(outputFile.exists());
writer.open(executionContext);
writer.close();
assertTrue(outputFile.exists());
}
@Test
public void testWriteHeaderAfterRestartOnFirstChunk() throws Exception {

157
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/xml/StaxEventItemWriterTests.java
View File

@@ -353,6 +353,163 @@ public class StaxEventItemWriterTests {
}
}
/**
* Resource is not deleted when items have been written and shouldDeleteIfEmpty flag is set.
*/
@Test
public void testDeleteIfEmptyRecordsWritten() throws Exception {
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.write(items);
writer.close();
String content = getOutputFileContent();
assertTrue("Wrong content: " + content, content.contains(TEST_STRING));
}
/**
* Resource is deleted when no items have been written and shouldDeleteIfEmpty flag is set.
*/
@Test
public void testDeleteIfEmptyNoRecordsWritten() throws Exception {
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.close();
assertFalse("file should be deleted" + resource, resource.getFile().exists());
}
/**
* Resource is deleted when items have not been written and shouldDeleteIfEmpty flag is set.
*/
@Test
public void testDeleteIfEmptyNoRecordsWrittenHeaderAndFooter() throws Exception {
writer.setShouldDeleteIfEmpty(true);
writer.setHeaderCallback(new StaxWriterCallback() {
@Override
public void write(XMLEventWriter writer) throws IOException {
XMLEventFactory factory = XMLEventFactory.newInstance();
try {
writer.add(factory.createStartElement("", "", "header"));
writer.add(factory.createEndElement("", "", "header"));
}
catch (XMLStreamException e) {
throw new RuntimeException(e);
}
}
});
writer.setFooterCallback(new StaxWriterCallback() {
@Override
public void write(XMLEventWriter writer) throws IOException {
XMLEventFactory factory = XMLEventFactory.newInstance();
try {
writer.add(factory.createStartElement("", "", "footer"));
writer.add(factory.createEndElement("", "", "footer"));
}
catch (XMLStreamException e) {
throw new RuntimeException(e);
}
}
});
writer.open(executionContext);
writer.close();
assertFalse("file should be deleted" + resource, resource.getFile().exists());
}
/**
* Resource is not deleted when items have been written and shouldDeleteIfEmpty flag is set.
*/
@Test
public void testDeleteIfEmptyRecordsWrittenRestart() throws Exception {
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.write(items);
writer.update(executionContext);
writer.close();
writer = createItemWriter();
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.close();
String content = getOutputFileContent();
assertTrue("Wrong content: " + content, content.contains(TEST_STRING));
}
/**
* Test that the writer can restart if the previous execution deleted empty file.
*/
@Test
public void testDeleteIfEmptyRestartAfterDelete() throws Exception {
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.update(executionContext);
writer.close();
assertFalse(resource.getFile().exists());
writer = createItemWriter();
writer.setShouldDeleteIfEmpty(true);
writer.open(executionContext);
writer.write(items);
writer.update(executionContext);
writer.close();
String content = getOutputFileContent();
assertTrue("Wrong content: " + content, content.contains(TEST_STRING));
}
/**
* Resource is not deleted when items have been written and shouldDeleteIfEmpty flag is set (restart after delete).
*/
@Test
public void testDeleteIfEmptyNoRecordsWrittenHeaderAndFooterRestartAfterDelete() throws Exception {
writer.setShouldDeleteIfEmpty(true);
writer.setHeaderCallback(new StaxWriterCallback() {
@Override
public void write(XMLEventWriter writer) throws IOException {
XMLEventFactory factory = XMLEventFactory.newInstance();
try {
writer.add(factory.createStartElement("", "", "header"));
writer.add(factory.createEndElement("", "", "header"));
}
catch (XMLStreamException e) {
throw new RuntimeException(e);
}
}
});
writer.setFooterCallback(new StaxWriterCallback() {
@Override
public void write(XMLEventWriter writer) throws IOException {
XMLEventFactory factory = XMLEventFactory.newInstance();
try {
writer.add(factory.createStartElement("", "", "footer"));
writer.add(factory.createEndElement("", "", "footer"));
}
catch (XMLStreamException e) {
throw new RuntimeException(e);
}
}
});
writer.open(executionContext);
writer.update(executionContext);
writer.close();
assertFalse("file should be deleted" + resource, resource.getFile().exists());
writer.open(executionContext);
writer.write(items);
writer.update(executionContext);
writer.close();
String content = getOutputFileContent();
assertTrue("Wrong content: " + content, content.contains(TEST_STRING));
}
/**
* Item is written to the output file with namespace.
*/