diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriter.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriter.java index f5562d47c..5074fd49b 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriter.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriter.java @@ -16,10 +16,12 @@ package org.springframework.batch.support.transaction; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.io.Writer; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import org.springframework.batch.item.WriteFailedException; import org.springframework.transaction.support.TransactionSynchronizationAdapter; import org.springframework.transaction.support.TransactionSynchronizationManager; @@ -137,13 +139,17 @@ public class TransactionAwareBufferedWriter extends Writer { * Convenience method for clients to determine if there is any unflushed * data. * - * @return the current size of unflushed buffered data + * @return the current size (in bytes) of unflushed buffered data */ public long getBufferSize() { if (!transactionActive()) { return 0L; } - return getCurrentBuffer().length(); + try { + return getCurrentBuffer().toString().getBytes(encoding).length; + } catch (UnsupportedEncodingException e) { + throw new WriteFailedException("Could not determine buffer size because of unsupported encoding: " + encoding, e); + } } /** diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemWriterTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemWriterTests.java index 22d125aba..4e9c0a764 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemWriterTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemWriterTests.java @@ -25,8 +25,9 @@ import static org.junit.Assert.fail; import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.io.Writer; import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; @@ -88,6 +89,7 @@ public class FlatFileItemWriterTests { writer.setLineAggregator(new PassThroughLineAggregator()); writer.afterPropertiesSet(); writer.setSaveState(true); + writer.setEncoding("UTF-8"); executionContext = new ExecutionContext(); } @@ -108,13 +110,22 @@ public class FlatFileItemWriterTests { * because running the tests in a UNIX environment locks the file if it's open for writing. */ private String readLine() throws IOException { + return readLine("UTF-8"); + } + + /* + * Read a line from the output file, if the reader has not been created, recreate. This method is only necessary + * because running the tests in a UNIX environment locks the file if it's open for writing. + */ + private String readLine(String encoding) throws IOException { if (reader == null) { - reader = new BufferedReader(new FileReader(outputFile)); + reader = new BufferedReader(new InputStreamReader(new FileInputStream(outputFile), encoding)); } return reader.readLine(); } + /* * Properly close the output file reader. */ @@ -231,7 +242,7 @@ public class FlatFileItemWriterTests { @Test public void testWriteWithConverter() throws Exception { writer.setLineAggregator(new LineAggregator() { - @Override + @Override public String aggregate(String item) { return "FOO:" + item; } @@ -252,7 +263,7 @@ public class FlatFileItemWriterTests { @Test public void testWriteWithConverterAndString() throws Exception { writer.setLineAggregator(new LineAggregator() { - @Override + @Override public String aggregate(String item) { return "FOO:" + item; } @@ -290,7 +301,7 @@ public class FlatFileItemWriterTests { writer.setFooterCallback(new FlatFileFooterCallback() { - @Override + @Override public void writeFooter(Writer writer) throws IOException { writer.write("footer"); } @@ -345,7 +356,7 @@ public class FlatFileItemWriterTests { writer.open(executionContext); new TransactionTemplate(transactionManager).execute(new TransactionCallback() { - @Override + @Override public Object doInTransaction(TransactionStatus status) { try { writer.write(Collections.singletonList(TEST_STRING)); @@ -366,7 +377,7 @@ public class FlatFileItemWriterTests { writer.setFooterCallback(new FlatFileFooterCallback() { - @Override + @Override public void writeFooter(Writer writer) throws IOException { writer.write("footer"); } @@ -378,7 +389,7 @@ public class FlatFileItemWriterTests { PlatformTransactionManager transactionManager = new ResourcelessTransactionManager(); new TransactionTemplate(transactionManager).execute(new TransactionCallback() { - @Override + @Override public Object doInTransaction(TransactionStatus status) { try { // write some lines @@ -401,7 +412,7 @@ public class FlatFileItemWriterTests { writer.open(executionContext); new TransactionTemplate(transactionManager).execute(new TransactionCallback() { - @Override + @Override public Object doInTransaction(TransactionStatus status) { try { // write more lines @@ -430,6 +441,85 @@ public class FlatFileItemWriterTests { } + @Test + // BATCH-1959 + public void testTransactionalRestartWithMultiByteCharacterUTF8() throws Exception { + testTransactionalRestartWithMultiByteCharacter("UTF-8"); + } + + @Test + // BATCH-1959 + public void testTransactionalRestartWithMultiByteCharacterUTF16BE() throws Exception { + testTransactionalRestartWithMultiByteCharacter("UTF-16BE"); + } + + private void testTransactionalRestartWithMultiByteCharacter(String encoding) throws Exception { + writer.setEncoding(encoding); + writer.setFooterCallback(new FlatFileFooterCallback() { + + @Override + public void writeFooter(Writer writer) throws IOException { + writer.write("footer"); + } + + }); + + writer.open(executionContext); + + PlatformTransactionManager transactionManager = new ResourcelessTransactionManager(); + + new TransactionTemplate(transactionManager).execute(new TransactionCallback() { + @Override + public Object doInTransaction(TransactionStatus status) { + try { + // write some lines + writer.write(Arrays.asList(new String[] { "téstLine1", "téstLine2", "téstLine3" })); + // write more lines + writer.write(Arrays.asList(new String[] { "téstLine4", "téstLine5" })); + } + catch (Exception e) { + throw new UnexpectedInputException("Could not write data", e); + } + // get restart data + writer.update(executionContext); + return null; + } + }); + // close template + writer.close(); + + // init with correct data + writer.open(executionContext); + + new TransactionTemplate(transactionManager).execute(new TransactionCallback() { + @Override + public Object doInTransaction(TransactionStatus status) { + try { + // write more lines + writer.write(Arrays.asList(new String[] { "téstLine6", "téstLine7", "téstLine8" })); + } + catch (Exception e) { + throw new UnexpectedInputException("Could not write data", e); + } + // get restart data + writer.update(executionContext); + return null; + } + }); + // close template + writer.close(); + + // verify what was written to the file + for (int i = 1; i <= 8; i++) { + assertEquals("téstLine" + i, readLine(encoding)); + } + + assertEquals("footer", readLine(encoding)); + + // 8 lines were written to the file in total + assertEquals(8, executionContext.getLong(ClassUtils.getShortName(FlatFileItemWriter.class) + ".written")); + } + @Test public void testOpenWithNonWritableFile() throws Exception { writer = new FlatFileItemWriter(); @@ -514,7 +604,7 @@ public class FlatFileItemWriterTests { public void testWriteFooter() throws Exception { writer.setFooterCallback(new FlatFileFooterCallback() { - @Override + @Override public void writeFooter(Writer writer) throws IOException { writer.write("a\nb"); } @@ -532,7 +622,7 @@ public class FlatFileItemWriterTests { public void testWriteHeader() throws Exception { writer.setHeaderCallback(new FlatFileHeaderCallback() { - @Override + @Override public void writeHeader(Writer writer) throws IOException { writer.write("a\nb"); } @@ -552,7 +642,7 @@ public class FlatFileItemWriterTests { @Test public void testWriteWithAppendAfterHeaders() throws Exception { writer.setHeaderCallback(new FlatFileHeaderCallback() { - @Override + @Override public void writeHeader(Writer writer) throws IOException { writer.write("a\nb"); } @@ -578,7 +668,7 @@ public class FlatFileItemWriterTests { public void testWriteHeaderAndDeleteOnExit() throws Exception { writer.setHeaderCallback(new FlatFileHeaderCallback() { - @Override + @Override public void writeHeader(Writer writer) throws IOException { writer.write("a\nb"); } @@ -603,12 +693,12 @@ public class FlatFileItemWriterTests { writer.write(Collections.singletonList("test2")); assertEquals("test2", readLine()); } - + @Test public void testWriteHeaderAndDeleteOnExitReopen() throws Exception { writer.setHeaderCallback(new FlatFileHeaderCallback() { - @Override + @Override public void writeHeader(Writer writer) throws IOException { writer.write("a\nb"); } @@ -626,8 +716,8 @@ public class FlatFileItemWriterTests { assertEquals("a", readLine()); assertEquals("b", readLine()); assertEquals("test2", readLine()); - } - + } + @Test public void testDeleteOnExitNoRecordsWrittenAfterRestart() throws Exception { writer.setShouldDeleteIfEmpty(true); @@ -645,7 +735,7 @@ public class FlatFileItemWriterTests { public void testWriteHeaderAfterRestartOnFirstChunk() throws Exception { writer.setHeaderCallback(new FlatFileHeaderCallback() { - @Override + @Override public void writeHeader(Writer writer) throws IOException { writer.write("a\nb"); } @@ -671,7 +761,7 @@ public class FlatFileItemWriterTests { public void testWriteHeaderAfterRestartOnSecondChunk() throws Exception { writer.setHeaderCallback(new FlatFileHeaderCallback() { - @Override + @Override public void writeHeader(Writer writer) throws IOException { writer.write("a\nb"); } @@ -710,7 +800,7 @@ public class FlatFileItemWriterTests { writer.setLineAggregator(new LineAggregator() { - @Override + @Override public String aggregate(String item) { if (item.equals("2")) { throw new RuntimeException("aggregation failed on " + item); diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/xml/StaxEventItemWriterTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/xml/StaxEventItemWriterTests.java index 5b10370c1..a467fdaf7 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/xml/StaxEventItemWriterTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/xml/StaxEventItemWriterTests.java @@ -60,13 +60,26 @@ public class StaxEventItemWriterTests { private JAXBItem jaxbItem = new JAXBItem(); + // test item for writing to output with multi byte character + private Object itemMultiByte = new Object() { + @Override + public String toString() { + return ClassUtils.getShortName(StaxEventItemWriter.class) + "-téstStrïng"; + } + }; + private List items = Collections.singletonList(item); + private List itemsMultiByte = Collections.singletonList(itemMultiByte); + private List jaxbItems = Collections.singletonList(jaxbItem); private static final String TEST_STRING = "<" + ClassUtils.getShortName(StaxEventItemWriter.class) + "-testString/>"; + private static final String TEST_STRING_MULTI_BYTE = "<" + ClassUtils.getShortName(StaxEventItemWriter.class) + + "-téstStrïng/>"; + private static final String NS_TEST_STRING = ""; @@ -185,6 +198,68 @@ public class StaxEventItemWriterTests { assertTrue(outputFile.contains("" + TEST_STRING + TEST_STRING + "")); } + @Test + // BATCH-1959 + public void testTransactionalRestartWithMultiByteCharacterUTF8() throws Exception { + testTransactionalRestartWithMultiByteCharacter("UTF-8"); + } + + @Test + // BATCH-1959 + public void testTransactionalRestartWithMultiByteCharacterUTF16BE() throws Exception { + testTransactionalRestartWithMultiByteCharacter("UTF-16BE"); + } + + private void testTransactionalRestartWithMultiByteCharacter(String encoding) throws Exception { + writer.setEncoding(encoding); + writer.open(executionContext); + + PlatformTransactionManager transactionManager = new ResourcelessTransactionManager(); + + new TransactionTemplate(transactionManager).execute(new TransactionCallback() { + @Override + public Object doInTransaction(TransactionStatus status) { + try { + // write item + writer.write(itemsMultiByte); + } + catch (Exception e) { + throw new UnexpectedInputException("Could not write data", e); + } + // get restart data + writer.update(executionContext); + return null; + } + }); + writer.close(); + + // create new writer from saved restart data and continue writing + writer = createItemWriter(); + writer.setEncoding(encoding); + writer.open(executionContext); + new TransactionTemplate(transactionManager).execute(new TransactionCallback() { + @Override + public Object doInTransaction(TransactionStatus status) { + try { + writer.write(itemsMultiByte); + } + catch (Exception e) { + throw new UnexpectedInputException("Could not write data", e); + } + // get restart data + writer.update(executionContext); + return null; + } + }); + writer.close(); + + // check the output is concatenation of 'before restart' and 'after + // restart' writes. + String outputFile = getOutputFileContent(encoding); + assertEquals(2, StringUtils.countOccurrencesOf(outputFile, TEST_STRING_MULTI_BYTE)); + assertTrue(outputFile.contains("" + TEST_STRING_MULTI_BYTE + TEST_STRING_MULTI_BYTE + "")); + } + @Test @SuppressWarnings({"unchecked", "rawtypes"}) public void testTransactionalRestartFailOnFirstWrite() throws Exception { @@ -692,11 +767,21 @@ public class StaxEventItemWriterTests { * @return output file content as String */ private String getOutputFileContent() throws IOException { - String value = FileUtils.readFileToString(resource.getFile(), null); - value = value.replace("", ""); + return getOutputFileContent("UTF-8"); + } + + + /** + * @param encoding the encoding + * @return output file content as String + */ + private String getOutputFileContent(String encoding) throws IOException { + String value = FileUtils.readFileToString(resource.getFile(), encoding); + value = value.replace("", ""); return value; } + /** * @return new instance of fully configured writer */ diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriterTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriterTests.java index e1e5c3a27..d562543e3 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriterTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriterTests.java @@ -63,6 +63,8 @@ public class TransactionAwareBufferedWriterTests { } } }); + + writer.setEncoding("UTF-8"); } private PlatformTransactionManager transactionManager = new ResourcelessTransactionManager(); @@ -100,7 +102,7 @@ public class TransactionAwareBufferedWriterTests { assertEquals(0, writer.getBufferSize()); } - + @Ignore //TODO - need to fix capture test @Test public void testCloseOutsideTransaction() throws Exception { @@ -183,6 +185,56 @@ public class TransactionAwareBufferedWriterTests { assertEquals(0, writer.getBufferSize()); } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + // BATCH-1959 + public void testBufferSizeInTransactionWithMultiByteCharacterUTF8() throws Exception { + ArgumentCaptor bb = ArgumentCaptor.forClass(ByteBuffer.class); + when(fileChannel.write(bb.capture())).thenReturn(5); + + new TransactionTemplate(transactionManager).execute(new TransactionCallback() { + @Override + public Object doInTransaction(TransactionStatus status) { + try { + writer.write("fóó"); + } + catch (IOException e) { + throw new IllegalStateException("Unexpected IOException", e); + } + assertEquals(5, writer.getBufferSize()); + return null; + } + }); + + assertEquals(0, writer.getBufferSize()); + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + // BATCH-1959 + public void testBufferSizeInTransactionWithMultiByteCharacterUTF16BE() throws Exception { + writer.setEncoding("UTF-16BE"); + + ArgumentCaptor bb = ArgumentCaptor.forClass(ByteBuffer.class); + when(fileChannel.write(bb.capture())).thenReturn(6); + + new TransactionTemplate(transactionManager).execute(new TransactionCallback() { + @Override + public Object doInTransaction(TransactionStatus status) { + try { + writer.write("fóó"); + } + catch (IOException e) { + throw new IllegalStateException("Unexpected IOException", e); + } + assertEquals(6, writer.getBufferSize()); + return null; + } + }); + + assertEquals(0, writer.getBufferSize()); + } @Test @SuppressWarnings({"unchecked", "rawtypes"})