Merge pull request #140 from jpraet/BATCH-1959

* BATCH-1959:
  BATCH-1959: Fixed whitespace issues
  fix assertion (see BATCH-1957: WRITTEN_STATISTICS_NAME in execution context now contains TOTAL number of records written)
  add additional tests with UTF-8 and UTF-16BE encoding
  BATCH-1959

spring-batch-infrastructure/src/main/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriter.java

@@ -16,10 +16,12 @@
 package org.springframework.batch.support.transaction;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.io.Writer;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 
+import org.springframework.batch.item.WriteFailedException;
 import org.springframework.transaction.support.TransactionSynchronizationAdapter;
 import org.springframework.transaction.support.TransactionSynchronizationManager;
 
@@ -137,13 +139,17 @@ public class TransactionAwareBufferedWriter extends Writer {
 	 * Convenience method for clients to determine if there is any unflushed
 	 * data.
 	 * 
-	 * @return the current size of unflushed buffered data
+	 * @return the current size (in bytes) of unflushed buffered data
 	 */
 	public long getBufferSize() {
 		if (!transactionActive()) {
 			return 0L;
 		}
-		return getCurrentBuffer().length();
+		try {			
+			return getCurrentBuffer().toString().getBytes(encoding).length;
+		} catch (UnsupportedEncodingException e) {
+			throw new WriteFailedException("Could not determine buffer size because of unsupported encoding: " + encoding, e);
+		}
 	}
 
 	/**

spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/FlatFileItemWriterTests.java

@@ -25,8 +25,9 @@ import static org.junit.Assert.fail;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Writer;
 import java.nio.charset.UnsupportedCharsetException;
 import java.util.ArrayList;
@@ -88,6 +89,7 @@ public class FlatFileItemWriterTests {
 		writer.setLineAggregator(new PassThroughLineAggregator<String>());
 		writer.afterPropertiesSet();
 		writer.setSaveState(true);
+		writer.setEncoding("UTF-8");
 		executionContext = new ExecutionContext();
 	}
 
@@ -108,13 +110,22 @@ public class FlatFileItemWriterTests {
 	 * because running the tests in a UNIX environment locks the file if it's open for writing.
 	 */
 	private String readLine() throws IOException {
+		return readLine("UTF-8");
+	}
+
+	/*
+	 * Read a line from the output file, if the reader has not been created, recreate. This method is only necessary
+	 * because running the tests in a UNIX environment locks the file if it's open for writing.
+	 */
+	private String readLine(String encoding) throws IOException {
 
 		if (reader == null) {
-			reader = new BufferedReader(new FileReader(outputFile));
+			reader = new BufferedReader(new InputStreamReader(new FileInputStream(outputFile), encoding));
 		}
 
 		return reader.readLine();
 	}
+
 	/*
 	 * Properly close the output file reader.
 	 */
@@ -231,7 +242,7 @@ public class FlatFileItemWriterTests {
 	@Test
 	public void testWriteWithConverter() throws Exception {
 		writer.setLineAggregator(new LineAggregator<String>() {
-            @Override
+			@Override
 			public String aggregate(String item) {
 				return "FOO:" + item;
 			}
@@ -252,7 +263,7 @@ public class FlatFileItemWriterTests {
 	@Test
 	public void testWriteWithConverterAndString() throws Exception {
 		writer.setLineAggregator(new LineAggregator<String>() {
-            @Override
+			@Override
 			public String aggregate(String item) {
 				return "FOO:" + item;
 			}
@@ -290,7 +301,7 @@ public class FlatFileItemWriterTests {
 
 		writer.setFooterCallback(new FlatFileFooterCallback() {
 
-            @Override
+			@Override
 			public void writeFooter(Writer writer) throws IOException {
 				writer.write("footer");
 			}
@@ -345,7 +356,7 @@ public class FlatFileItemWriterTests {
 
 		writer.open(executionContext);
 		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
-            @Override
+			@Override
 			public Object doInTransaction(TransactionStatus status) {
 				try {
 					writer.write(Collections.singletonList(TEST_STRING));
@@ -366,7 +377,7 @@ public class FlatFileItemWriterTests {
 
 		writer.setFooterCallback(new FlatFileFooterCallback() {
 
-            @Override
+			@Override
 			public void writeFooter(Writer writer) throws IOException {
 				writer.write("footer");
 			}
@@ -378,7 +389,7 @@ public class FlatFileItemWriterTests {
 		PlatformTransactionManager transactionManager = new ResourcelessTransactionManager();
 
 		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
-            @Override
+			@Override
 			public Object doInTransaction(TransactionStatus status) {
 				try {
 					// write some lines
@@ -401,7 +412,7 @@ public class FlatFileItemWriterTests {
 		writer.open(executionContext);
 
 		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
-            @Override
+			@Override
 			public Object doInTransaction(TransactionStatus status) {
 				try {
 					// write more lines
@@ -430,6 +441,85 @@ public class FlatFileItemWriterTests {
 
 	}
 
+	@Test
+	// BATCH-1959
+	public void testTransactionalRestartWithMultiByteCharacterUTF8() throws Exception {
+		testTransactionalRestartWithMultiByteCharacter("UTF-8");
+	}
+
+	@Test
+	// BATCH-1959
+	public void testTransactionalRestartWithMultiByteCharacterUTF16BE() throws Exception {
+		testTransactionalRestartWithMultiByteCharacter("UTF-16BE");
+	}
+
+	private void testTransactionalRestartWithMultiByteCharacter(String encoding) throws Exception {
+		writer.setEncoding(encoding);
+		writer.setFooterCallback(new FlatFileFooterCallback() {
+
+			@Override
+			public void writeFooter(Writer writer) throws IOException {
+				writer.write("footer");
+			}
+
+		});
+
+		writer.open(executionContext);
+
+		PlatformTransactionManager transactionManager = new ResourcelessTransactionManager();
+
+		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
+			@Override
+			public Object doInTransaction(TransactionStatus status) {
+				try {
+					// write some lines
+					writer.write(Arrays.asList(new String[] { "téstLine1", "téstLine2", "téstLine3" }));
+					// write more lines
+					writer.write(Arrays.asList(new String[] { "téstLine4", "téstLine5" }));
+				}
+				catch (Exception e) {
+					throw new UnexpectedInputException("Could not write data", e);
+				}
+				// get restart data
+				writer.update(executionContext);
+				return null;
+			}
+		});
+		// close template
+		writer.close();
+
+		// init with correct data
+		writer.open(executionContext);
+
+		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
+			@Override
+			public Object doInTransaction(TransactionStatus status) {
+				try {
+					// write more lines
+					writer.write(Arrays.asList(new String[] { "téstLine6", "téstLine7", "téstLine8" }));
+				}
+				catch (Exception e) {
+					throw new UnexpectedInputException("Could not write data", e);
+				}
+				// get restart data
+				writer.update(executionContext);
+				return null;
+			}
+		});
+		// close template
+		writer.close();
+
+		// verify what was written to the file
+		for (int i = 1; i <= 8; i++) {
+			assertEquals("téstLine" + i, readLine(encoding));
+		}
+
+		assertEquals("footer", readLine(encoding));
+
+		// 8 lines were written to the file in total
+		assertEquals(8, executionContext.getLong(ClassUtils.getShortName(FlatFileItemWriter.class) + ".written"));
+	}
+
 	@Test
 	public void testOpenWithNonWritableFile() throws Exception {
 		writer = new FlatFileItemWriter<String>();
@@ -514,7 +604,7 @@ public class FlatFileItemWriterTests {
 	public void testWriteFooter() throws Exception {
 		writer.setFooterCallback(new FlatFileFooterCallback() {
 
-            @Override
+			@Override
 			public void writeFooter(Writer writer) throws IOException {
 				writer.write("a\nb");
 			}
@@ -532,7 +622,7 @@ public class FlatFileItemWriterTests {
 	public void testWriteHeader() throws Exception {
 		writer.setHeaderCallback(new FlatFileHeaderCallback() {
 
-            @Override
+			@Override
 			public void writeHeader(Writer writer) throws IOException {
 				writer.write("a\nb");
 			}
@@ -552,7 +642,7 @@ public class FlatFileItemWriterTests {
 	@Test
 	public void testWriteWithAppendAfterHeaders() throws Exception {
 		writer.setHeaderCallback(new FlatFileHeaderCallback() {
-            @Override
+			@Override
 			public void writeHeader(Writer writer) throws IOException {
 				writer.write("a\nb");
 			}
@@ -578,7 +668,7 @@ public class FlatFileItemWriterTests {
 	public void testWriteHeaderAndDeleteOnExit() throws Exception {
 		writer.setHeaderCallback(new FlatFileHeaderCallback() {
 
-            @Override
+			@Override
 			public void writeHeader(Writer writer) throws IOException {
 				writer.write("a\nb");
 			}
@@ -603,12 +693,12 @@ public class FlatFileItemWriterTests {
 		writer.write(Collections.singletonList("test2"));
 		assertEquals("test2", readLine());
 	}
-	
+
 	@Test
 	public void testWriteHeaderAndDeleteOnExitReopen() throws Exception {
 		writer.setHeaderCallback(new FlatFileHeaderCallback() {
 
-            @Override
+			@Override
 			public void writeHeader(Writer writer) throws IOException {
 				writer.write("a\nb");
 			}
@@ -626,8 +716,8 @@ public class FlatFileItemWriterTests {
 		assertEquals("a", readLine());
 		assertEquals("b", readLine());
 		assertEquals("test2", readLine());
-	}	
-	
+	}
+
 	@Test
 	public void testDeleteOnExitNoRecordsWrittenAfterRestart() throws Exception {
 		writer.setShouldDeleteIfEmpty(true);
@@ -645,7 +735,7 @@ public class FlatFileItemWriterTests {
 	public void testWriteHeaderAfterRestartOnFirstChunk() throws Exception {
 		writer.setHeaderCallback(new FlatFileHeaderCallback() {
 
-            @Override
+			@Override
 			public void writeHeader(Writer writer) throws IOException {
 				writer.write("a\nb");
 			}
@@ -671,7 +761,7 @@ public class FlatFileItemWriterTests {
 	public void testWriteHeaderAfterRestartOnSecondChunk() throws Exception {
 		writer.setHeaderCallback(new FlatFileHeaderCallback() {
 
-            @Override
+			@Override
 			public void writeHeader(Writer writer) throws IOException {
 				writer.write("a\nb");
 			}
@@ -710,7 +800,7 @@ public class FlatFileItemWriterTests {
 
 		writer.setLineAggregator(new LineAggregator<String>() {
 
-            @Override
+			@Override
 			public String aggregate(String item) {
 				if (item.equals("2")) {
 					throw new RuntimeException("aggregation failed on " + item);

spring-batch-infrastructure/src/test/java/org/springframework/batch/item/xml/StaxEventItemWriterTests.java

@@ -60,13 +60,26 @@ public class StaxEventItemWriterTests {
 
 	private JAXBItem jaxbItem = new JAXBItem();
 
+	// test item for writing to output with multi byte character
+	private Object itemMultiByte = new Object() {
+		@Override
+		public String toString() {
+			return ClassUtils.getShortName(StaxEventItemWriter.class) + "-téstStrïng";
+		}
+	};
+
 	private List<? extends Object> items = Collections.singletonList(item);
 
+	private List<? extends Object> itemsMultiByte = Collections.singletonList(itemMultiByte);
+
 	private List<? extends Object> jaxbItems = Collections.singletonList(jaxbItem);
 
 	private static final String TEST_STRING = "<" + ClassUtils.getShortName(StaxEventItemWriter.class)
 			+ "-testString/>";
 
+	private static final String TEST_STRING_MULTI_BYTE = "<" + ClassUtils.getShortName(StaxEventItemWriter.class)
+			+ "-téstStrïng/>";
+
 	private static final String NS_TEST_STRING = "<ns:" + ClassUtils.getShortName(StaxEventItemWriter.class)
 			+ "-testString/>";
 
@@ -185,6 +198,68 @@ public class StaxEventItemWriterTests {
 		assertTrue(outputFile.contains("<root>" + TEST_STRING + TEST_STRING + "</root>"));
 	}
 
+	@Test
+	// BATCH-1959
+	public void testTransactionalRestartWithMultiByteCharacterUTF8() throws Exception {
+		testTransactionalRestartWithMultiByteCharacter("UTF-8");
+	}
+
+	@Test
+	// BATCH-1959
+	public void testTransactionalRestartWithMultiByteCharacterUTF16BE() throws Exception {
+		testTransactionalRestartWithMultiByteCharacter("UTF-16BE");
+	}
+
+	private void testTransactionalRestartWithMultiByteCharacter(String encoding) throws Exception {
+		writer.setEncoding(encoding);
+		writer.open(executionContext);
+
+		PlatformTransactionManager transactionManager = new ResourcelessTransactionManager();
+
+		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
+			@Override
+			public Object doInTransaction(TransactionStatus status) {
+				try {
+					// write item
+					writer.write(itemsMultiByte);
+				}
+				catch (Exception e) {
+					throw new UnexpectedInputException("Could not write data", e);
+				}
+				// get restart data
+				writer.update(executionContext);
+				return null;
+			}
+		});
+		writer.close();
+
+		// create new writer from saved restart data and continue writing
+		writer = createItemWriter();
+		writer.setEncoding(encoding);
+		writer.open(executionContext);
+		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
+			@Override
+			public Object doInTransaction(TransactionStatus status) {
+				try {
+					writer.write(itemsMultiByte);
+				}
+				catch (Exception e) {
+					throw new UnexpectedInputException("Could not write data", e);
+				}
+				// get restart data
+				writer.update(executionContext);
+				return null;
+			}
+		});
+		writer.close();
+
+		// check the output is concatenation of 'before restart' and 'after
+		// restart' writes.
+		String outputFile = getOutputFileContent(encoding);
+		assertEquals(2, StringUtils.countOccurrencesOf(outputFile, TEST_STRING_MULTI_BYTE));
+		assertTrue(outputFile.contains("<root>" + TEST_STRING_MULTI_BYTE + TEST_STRING_MULTI_BYTE + "</root>"));
+	}
+
 	@Test
 	@SuppressWarnings({"unchecked", "rawtypes"})
 	public void testTransactionalRestartFailOnFirstWrite() throws Exception {
@@ -692,11 +767,21 @@ public class StaxEventItemWriterTests {
 	 * @return output file content as String
 	 */
 	private String getOutputFileContent() throws IOException {
-		String value = FileUtils.readFileToString(resource.getFile(), null);
-		value = value.replace("<?xml version='1.0' encoding='UTF-8'?>", "");
+		return getOutputFileContent("UTF-8");
+	}
+
+
+	/**
+	 * @param encoding the encoding
+	 * @return output file content as String
+	 */
+	private String getOutputFileContent(String encoding) throws IOException {
+		String value = FileUtils.readFileToString(resource.getFile(), encoding);
+		value = value.replace("<?xml version='1.0' encoding='" + encoding + "'?>", "");
 		return value;
 	}
 
+
 	/**
 	 * @return new instance of fully configured writer
 	 */

spring-batch-infrastructure/src/test/java/org/springframework/batch/support/transaction/TransactionAwareBufferedWriterTests.java

@@ -63,6 +63,8 @@ public class TransactionAwareBufferedWriterTests {
 				}
 			}
 		});
+		
+		writer.setEncoding("UTF-8");
 	}
 
 	private PlatformTransactionManager transactionManager = new ResourcelessTransactionManager();
@@ -100,7 +102,7 @@ public class TransactionAwareBufferedWriterTests {
 		
 		assertEquals(0, writer.getBufferSize());
 	}
-
+	
 	@Ignore //TODO - need to fix capture test
 	@Test
 	public void testCloseOutsideTransaction() throws Exception {
@@ -183,6 +185,56 @@ public class TransactionAwareBufferedWriterTests {
 		
 		assertEquals(0, writer.getBufferSize());
 	}
+	
+	@Test
+	@SuppressWarnings({"unchecked", "rawtypes"})
+	// BATCH-1959
+	public void testBufferSizeInTransactionWithMultiByteCharacterUTF8() throws Exception {
+		ArgumentCaptor<ByteBuffer> bb = ArgumentCaptor.forClass(ByteBuffer.class);
+		when(fileChannel.write(bb.capture())).thenReturn(5);
+
+		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
+            @Override
+			public Object doInTransaction(TransactionStatus status) {
+				try {
+					writer.write("fóó");
+				}
+				catch (IOException e) {
+					throw new IllegalStateException("Unexpected IOException", e);
+				}
+				assertEquals(5, writer.getBufferSize());
+				return null;
+			}
+		});
+		
+		assertEquals(0, writer.getBufferSize());
+	}	
+
+	@Test
+	@SuppressWarnings({"unchecked", "rawtypes"})
+	// BATCH-1959
+	public void testBufferSizeInTransactionWithMultiByteCharacterUTF16BE() throws Exception {
+		writer.setEncoding("UTF-16BE");
+		
+		ArgumentCaptor<ByteBuffer> bb = ArgumentCaptor.forClass(ByteBuffer.class);
+		when(fileChannel.write(bb.capture())).thenReturn(6);
+
+		new TransactionTemplate(transactionManager).execute(new TransactionCallback() {
+            @Override
+			public Object doInTransaction(TransactionStatus status) {
+				try {
+					writer.write("fóó");
+				}
+				catch (IOException e) {
+					throw new IllegalStateException("Unexpected IOException", e);
+				}
+				assertEquals(6, writer.getBufferSize());
+				return null;
+			}
+		});
+		
+		assertEquals(0, writer.getBufferSize());
+	}	
 
 	@Test
 	@SuppressWarnings({"unchecked", "rawtypes"})