Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor MultiResourceItemWriter

Browse Source 
The `MultiResourceItemWriter` now writes at most `itemCountLimitPerResource`
items per resource where it previously allowed more items when they were
written within the same chunk.

Resolves #1722
This commit is contained in:
Henning Poettker
2025-01-07 22:16:43 +01:00
committed by Mahmoud Ben Hassine
parent 3b0868db5d
commit f758d14ee6
4 changed files with 150 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/MultiResourceItemWriter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2006-2024 the original author or authors.
* Copyright 2006-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -34,9 +34,6 @@ import org.springframework.util.ClassUtils;
* {@link #setItemCountLimitPerResource(int)}. Suffix creation can be customized with
* {@link #setResourceSuffixCreator(ResourceSuffixCreator)}.
* <p>
* Note that new resources are created only at chunk boundaries i.e. the number of items
* written into one resource is between the limit set by
* <p>
* This writer will create an output file only when there are items to write, which means
* there would be no empty file created if no items are passed (for example when all items
* are filtered or skipped during the processing phase).
@@ -45,6 +42,7 @@ import org.springframework.util.ClassUtils;
* @param <T> item type
* @author Robert Kasanicky
* @author Mahmoud Ben Hassine
* @author Henning Pöttker
*/
public class MultiResourceItemWriter<T> extends AbstractItemStreamItemWriter<T> {
@@ -74,22 +72,30 @@ public class MultiResourceItemWriter<T> extends AbstractItemStreamItemWriter<T>
@Override
public void write(Chunk<? extends T> items) throws Exception {
if (!opened) {
File file = setResourceToDelegate();
// create only if write is called
file.createNewFile();
Assert.state(file.canWrite(), "Output resource " + file.getAbsolutePath() + " must be writable");
delegate.open(new ExecutionContext());
opened = true;
}
delegate.write(items);
currentResourceItemCount += items.size();
if (currentResourceItemCount >= itemCountLimitPerResource) {
delegate.close();
resourceIndex++;
currentResourceItemCount = 0;
setResourceToDelegate();
opened = false;
int writtenItems = 0;
while (writtenItems < items.size()) {
if (!opened) {
File file = setResourceToDelegate();
// create only if write is called
file.createNewFile();
Assert.state(file.canWrite(), "Output resource " + file.getAbsolutePath() + " must be writable");
delegate.open(new ExecutionContext());
opened = true;
}
int itemsToWrite = Math.min(itemCountLimitPerResource - currentResourceItemCount,
items.size() - writtenItems);
delegate.write(new Chunk<T>(items.getItems().subList(writtenItems, writtenItems + itemsToWrite)));
currentResourceItemCount += itemsToWrite;
writtenItems += itemsToWrite;
if (currentResourceItemCount >= itemCountLimitPerResource) {
delegate.close();
resourceIndex++;
currentResourceItemCount = 0;
setResourceToDelegate();
opened = false;
}
}
}

129
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/MultiResourceItemWriterFlatFileTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2008-2023 the original author or authors.
* Copyright 2008-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -78,22 +78,22 @@ public class MultiResourceItemWriterFlatFileTests extends AbstractMultiResourceI
tested.write(Chunk.of("1", "2", "3"));
File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertEquals("123", readFile(part1));
assertFileExistsAndContains(1, "12");
assertFileExistsAndContains(2, "3");
tested.write(Chunk.of("4"));
File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertEquals("4", readFile(part2));
assertFileExistsAndContains(2, "34");
tested.write(Chunk.of("5"));
assertEquals("45", readFile(part2));
assertFileExistsAndContains(3, "5");
tested.write(Chunk.of("6", "7", "8", "9"));
File part3 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(3));
assertTrue(part3.exists());
assertEquals("6789", readFile(part3));
assertFileExistsAndContains(3, "56");
assertFileExistsAndContains(4, "78");
assertFileExistsAndContains(5, "9");
}
@Test
@@ -107,7 +107,7 @@ public class MultiResourceItemWriterFlatFileTests extends AbstractMultiResourceI
assertEquals(1, executionContext.getInt(tested.getExecutionContextKey("resource.index")));
tested.write(Chunk.of("1", "2", "3"));
tested.update(executionContext);
assertEquals(0, executionContext.getInt(tested.getExecutionContextKey("resource.item.count")));
assertEquals(1, executionContext.getInt(tested.getExecutionContextKey("resource.item.count")));
assertEquals(2, executionContext.getInt(tested.getExecutionContextKey("resource.index")));
}
@@ -121,17 +121,22 @@ public class MultiResourceItemWriterFlatFileTests extends AbstractMultiResourceI
tested.write(Chunk.of("1", "2", "3"));
File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertFileExistsAndContains(1, "12f");
assertFileExistsAndContains(2, "3");
tested.write(Chunk.of("4"));
File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertFileExistsAndContains(2, "34f");
tested.write(Chunk.of("5"));
assertFileExistsAndContains(3, "5");
tested.close();
assertEquals("123f", readFile(part1));
assertEquals("4f", readFile(part2));
assertFileExistsAndContains(1, "12f");
assertFileExistsAndContains(2, "34f");
assertFileExistsAndContains(3, "5f");
}
@@ -144,19 +149,18 @@ public class MultiResourceItemWriterFlatFileTests extends AbstractMultiResourceI
ResourcelessTransactionManager transactionManager = new ResourcelessTransactionManager();
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("1", "2", "3")));
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("1", "2")));
File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertFileExistsAndContains(1, "12f");
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("4")));
File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
assertTrue(part2.exists());
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("3")));
assertFileExistsAndContains(2, "3");
tested.close();
assertEquals("123f", readFile(part1));
assertEquals("4f", readFile(part2));
assertFileExistsAndContains(1, "12f");
assertFileExistsAndContains(2, "3f");
}
@@ -168,27 +172,23 @@ public class MultiResourceItemWriterFlatFileTests extends AbstractMultiResourceI
tested.write(Chunk.of("1", "2", "3"));
File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertEquals("123", readFile(part1));
tested.write(Chunk.of("4"));
File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertEquals("4", readFile(part2));
assertFileExistsAndContains(1, "12");
assertFileExistsAndContains(2, "3");
tested.update(executionContext);
tested.close();
tested.open(executionContext);
tested.write(Chunk.of("5"));
assertEquals("45", readFile(part2));
tested.write(Chunk.of("4"));
tested.write(Chunk.of("6", "7", "8", "9"));
File part3 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(3));
assertTrue(part3.exists());
assertEquals("6789", readFile(part3));
assertFileExistsAndContains(2, "34");
tested.write(Chunk.of("5", "6", "7", "8", "9"));
assertFileExistsAndContains(3, "56");
assertFileExistsAndContains(4, "78");
assertFileExistsAndContains(5, "9");
}
@Test
@@ -201,27 +201,24 @@ public class MultiResourceItemWriterFlatFileTests extends AbstractMultiResourceI
tested.write(Chunk.of("1", "2", "3"));
File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertEquals("123f", readFile(part1));
tested.write(Chunk.of("4"));
File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertEquals("4", readFile(part2));
assertFileExistsAndContains(1, "12f");
assertFileExistsAndContains(2, "3");
tested.update(executionContext);
tested.close();
tested.open(executionContext);
tested.write(Chunk.of("5"));
assertEquals("45f", readFile(part2));
tested.write(Chunk.of("4"));
tested.write(Chunk.of("6", "7", "8", "9"));
File part3 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(3));
assertTrue(part3.exists());
assertEquals("6789f", readFile(part3));
assertFileExistsAndContains(2, "34f");
tested.write(Chunk.of("5", "6", "7", "8", "9"));
tested.close();
assertFileExistsAndContains(3, "56f");
assertFileExistsAndContains(4, "78f");
assertFileExistsAndContains(5, "9f");
}
@Test
@@ -233,24 +230,28 @@ public class MultiResourceItemWriterFlatFileTests extends AbstractMultiResourceI
ResourcelessTransactionManager transactionManager = new ResourcelessTransactionManager();
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("1", "2", "3")));
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("1", "2")));
File part1 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertEquals("123f", readFile(part1));
assertFileExistsAndContains(1, "12f");
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("4")));
File part2 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertEquals("4", readFile(part2));
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("3")));
assertFileExistsAndContains(2, "3");
tested.update(executionContext);
tested.close();
tested.open(executionContext);
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("5")));
assertEquals("45f", readFile(part2));
new TransactionTemplate(transactionManager).execute(new WriterCallback(Chunk.of("4")));
assertFileExistsAndContains(2, "34f");
}
private void assertFileExistsAndContains(int index, String expected) throws Exception {
File part = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(index));
assertTrue(part.exists());
assertEquals(expected, readFile(part));
}
}

19
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/MultiResourceItemWriterXmlTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2009-2022 the original author or authors.
* Copyright 2009-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -108,21 +108,26 @@ class MultiResourceItemWriterXmlTests extends AbstractMultiResourceItemWriterTes
tested.update(executionContext);
tested.close();
assertEquals(xmlDocStart + "<prefix:4/>" + xmlDocEnd, readFile(part2));
assertEquals(xmlDocStart + "<prefix:1/><prefix:2/><prefix:3/>" + xmlDocEnd, readFile(part1));
assertEquals(xmlDocStart + "<prefix:3/><prefix:4/>" + xmlDocEnd, readFile(part2));
assertEquals(xmlDocStart + "<prefix:1/><prefix:2/>" + xmlDocEnd, readFile(part1));
tested.open(executionContext);
tested.write(Chunk.of("5"));
tested.write(Chunk.of("6", "7", "8", "9"));
File part3 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(3));
assertTrue(part3.exists());
tested.write(Chunk.of("6", "7", "8", "9"));
File part4 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(4));
assertTrue(part4.exists());
File part5 = new File(file.getAbsolutePath() + suffixCreator.getSuffix(5));
assertTrue(part5.exists());
tested.close();
assertEquals(xmlDocStart + "<prefix:4/><prefix:5/>" + xmlDocEnd, readFile(part2));
assertEquals(xmlDocStart + "<prefix:6/><prefix:7/><prefix:8/><prefix:9/>" + xmlDocEnd, readFile(part3));
assertEquals(xmlDocStart + "<prefix:5/><prefix:6/>" + xmlDocEnd, readFile(part3));
assertEquals(xmlDocStart + "<prefix:7/><prefix:8/>" + xmlDocEnd, readFile(part4));
assertEquals(xmlDocStart + "<prefix:9/>" + xmlDocEnd, readFile(part5));
}
}

93
spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/builder/MultiResourceItemWriterBuilderTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2017-2023 the original author or authors.
* Copyright 2017-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -84,22 +84,22 @@ class MultiResourceItemWriterBuilderTests {
this.writer.write(Chunk.of("1", "2", "3"));
File part1 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertEquals("123", readFile(part1));
assertFileExistsAndContains(1, "12");
assertFileExistsAndContains(2, "3");
this.writer.write(Chunk.of("4"));
File part2 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertEquals("4", readFile(part2));
assertFileExistsAndContains(2, "34");
this.writer.write(Chunk.of("5"));
assertEquals("45", readFile(part2));
assertFileExistsAndContains(3, "5");
this.writer.write(Chunk.of("6", "7", "8", "9"));
File part3 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(3));
assertTrue(part3.exists());
assertEquals("6789", readFile(part3));
assertFileExistsAndContains(3, "56");
assertFileExistsAndContains(4, "78");
assertFileExistsAndContains(5, "9");
}
@Test
@@ -117,14 +117,12 @@ class MultiResourceItemWriterBuilderTests {
this.writer.write(Chunk.of("1", "2", "3"));
File part1 = new File(this.file.getAbsolutePath() + simpleResourceSuffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertEquals("123", readFile(part1));
assertFileExistsAndContains(1, "12", simpleResourceSuffixCreator);
assertFileExistsAndContains(2, "3", simpleResourceSuffixCreator);
this.writer.write(Chunk.of("4"));
File part2 = new File(this.file.getAbsolutePath() + simpleResourceSuffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertEquals("4", readFile(part2));
assertFileExistsAndContains(2, "34", simpleResourceSuffixCreator);
}
@Test
@@ -143,7 +141,7 @@ class MultiResourceItemWriterBuilderTests {
assertEquals(1, this.executionContext.getInt(this.writer.getExecutionContextKey("resource.index")));
this.writer.write(Chunk.of("1", "2", "3"));
this.writer.update(this.executionContext);
assertEquals(0, this.executionContext.getInt(this.writer.getExecutionContextKey("resource.item.count")));
assertEquals(1, this.executionContext.getInt(this.writer.getExecutionContextKey("resource.item.count")));
assertEquals(2, this.executionContext.getInt(this.writer.getExecutionContextKey("resource.index")));
}
@@ -160,26 +158,21 @@ class MultiResourceItemWriterBuilderTests {
this.writer.write(Chunk.of("1", "2", "3"));
File part1 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertEquals("123", readFile(part1));
this.writer.write(Chunk.of("4"));
File part2 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertEquals("4", readFile(part2));
assertFileExistsAndContains(1, "12");
assertFileExistsAndContains(2, "3");
this.writer.update(this.executionContext);
this.writer.close();
this.writer.open(this.executionContext);
this.writer.write(Chunk.of("5"));
assertEquals("45", readFile(part2));
this.writer.write(Chunk.of("4"));
this.writer.write(Chunk.of("6", "7", "8", "9"));
File part3 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(3));
assertTrue(part3.exists());
assertEquals("6789", readFile(part3));
assertFileExistsAndContains(2, "34");
this.writer.write(Chunk.of("5", "6", "7", "8"));
assertFileExistsAndContains(3, "56");
assertFileExistsAndContains(4, "78");
}
@Test
@@ -195,26 +188,23 @@ class MultiResourceItemWriterBuilderTests {
this.writer.write(Chunk.of("1", "2", "3"));
File part1 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(1));
assertTrue(part1.exists());
assertEquals("123", readFile(part1));
this.writer.write(Chunk.of("4"));
File part2 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(2));
assertTrue(part2.exists());
assertEquals("4", readFile(part2));
assertFileExistsAndContains(1, "12");
assertFileExistsAndContains(2, "3");
this.writer.update(this.executionContext);
this.writer.close();
this.writer.open(this.executionContext);
this.writer.write(Chunk.of("5"));
assertEquals("4", readFile(part2));
this.writer.write(Chunk.of("4"));
this.writer.write(Chunk.of("6", "7", "8", "9"));
File part3 = new File(this.file.getAbsolutePath() + this.suffixCreator.getSuffix(1));
assertTrue(part3.exists());
assertEquals("56789", readFile(part3));
assertFileExistsAndContains(2, "3");
assertFileExistsAndContains(1, "4");
this.writer.write(Chunk.of("5", "6", "7", "8"));
assertFileExistsAndContains(1, "45");
assertFileExistsAndContains(2, "67");
assertFileExistsAndContains(3, "8");
}
@Test
@@ -265,4 +255,15 @@ class MultiResourceItemWriterBuilderTests {
return result.toString();
}
private void assertFileExistsAndContains(int index, String expected) throws Exception {
assertFileExistsAndContains(index, expected, this.suffixCreator);
}
private void assertFileExistsAndContains(int index, String expected, ResourceSuffixCreator suffixCreator)
throws Exception {
File part = new File(this.file.getAbsolutePath() + suffixCreator.getSuffix(index));
assertTrue(part.exists());
assertEquals(expected, readFile(part));
}
}