From 2cc807de6237d48cd6b6f14f7ed9a29a4530ab17 Mon Sep 17 00:00:00 2001 From: Mahmoud Ben Hassine Date: Thu, 4 Feb 2021 16:01:22 +0100 Subject: [PATCH] Fix tests failing on Windows Before this commit, tests in this change set were failing on MS Windows due to the file comparison method which was based on file content hash comparison. With this method, differences between OS line endings (LF vs CRLF) in generated json files produce different hashes. This commit uses a logical json comparison based on the jsonassert library. --- pom.xml | 1 + spring-batch-infrastructure/pom.xml | 6 + ...GsonJsonFileItemWriterFunctionalTests.java | 7 +- ...ksonJsonFileItemWriterFunctionalTests.java | 7 +- .../JsonFileItemWriterFunctionalTests.java | 261 +++++++++++------- 5 files changed, 181 insertions(+), 101 deletions(-) diff --git a/pom.xml b/pom.xml index 163f736dd..df781bafb 100644 --- a/pom.xml +++ b/pom.xml @@ -105,6 +105,7 @@ 1.0 1 1.2 + 1.5.0 5.4.24.Final diff --git a/spring-batch-infrastructure/pom.xml b/spring-batch-infrastructure/pom.xml index f95988aea..d5dda0941 100644 --- a/spring-batch-infrastructure/pom.xml +++ b/spring-batch-infrastructure/pom.xml @@ -211,6 +211,12 @@ ${xmlunit.version} test + + org.skyscreamer + jsonassert + ${jsonassert.version} + test + commons-io commons-io diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/GsonJsonFileItemWriterFunctionalTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/GsonJsonFileItemWriterFunctionalTests.java index bcd6b4e7a..da4ed68d4 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/GsonJsonFileItemWriterFunctionalTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/GsonJsonFileItemWriterFunctionalTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,4 +44,9 @@ public class GsonJsonFileItemWriterFunctionalTests extends JsonFileItemWriterFun return "expected-trades-gson-pretty-print.json"; } + @Override + protected String getMarshallerName() { + return "gson"; + } + } diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/JacksonJsonFileItemWriterFunctionalTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/JacksonJsonFileItemWriterFunctionalTests.java index d800bc04a..a4a16652a 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/JacksonJsonFileItemWriterFunctionalTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/JacksonJsonFileItemWriterFunctionalTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,4 +45,9 @@ public class JacksonJsonFileItemWriterFunctionalTests extends JsonFileItemWriter return "expected-trades-jackson-pretty-print.json"; } + @Override + protected String getMarshallerName() { + return "jackson"; + } + } diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/JsonFileItemWriterFunctionalTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/JsonFileItemWriterFunctionalTests.java index fb3730479..c6a915585 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/JsonFileItemWriterFunctionalTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/json/JsonFileItemWriterFunctionalTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 the original author or authors. + * Copyright 2018-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,9 @@ package org.springframework.batch.item.json; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.math.BigDecimal; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -29,6 +31,7 @@ import java.util.List; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.skyscreamer.jsonassert.JSONAssert; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.UnexpectedInputException; @@ -54,9 +57,6 @@ public abstract class JsonFileItemWriterFunctionalTests { private static final String EXPECTED_FILE_DIRECTORY = "src/test/resources/org/springframework/batch/item/json/"; - private Resource resource; - private List items; - private ExecutionContext executionContext; private Trade trade1 = new Trade("123", 5, new BigDecimal("10.5"), "foo"); private Trade trade2 = new Trade("456", 10, new BigDecimal("20.5"), "bar"); private Trade trade3 = new Trade("789", 15, new BigDecimal("30.5"), "foobar"); @@ -65,217 +65,274 @@ public abstract class JsonFileItemWriterFunctionalTests { protected abstract JsonObjectMarshaller getJsonObjectMarshaller(); protected abstract JsonObjectMarshaller getJsonObjectMarshallerWithPrettyPrint(); protected abstract String getExpectedPrettyPrintedFile(); - - private JsonFileItemWriter writer; - - @Before - public void setUp() throws Exception { - Path outputFilePath = Paths.get("target", "trades.json"); - Files.deleteIfExists(outputFilePath); - this.resource = new FileSystemResource(outputFilePath.toFile()); - this.executionContext = new ExecutionContext(); - this.items = Arrays.asList(this.trade1, this.trade2); - this.writer = new JsonFileItemWriterBuilder() - .name("tradesItemWriter") - .resource(this.resource) - .jsonObjectMarshaller(getJsonObjectMarshaller()) - .build(); - } + protected abstract String getMarshallerName(); @Test public void testJsonWriting() throws Exception { + //given + Path outputFilePath = Paths.get("target", "trades-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(resource) + .jsonObjectMarshaller(getJsonObjectMarshaller()) + .build(); + // when - this.writer.open(this.executionContext); - this.writer.write(this.items); - this.writer.close(); + writer.open(new ExecutionContext()); + writer.write(Arrays.asList(this.trade1, this.trade2)); + writer.close(); // then assertFileEquals( new File(EXPECTED_FILE_DIRECTORY + "expected-trades.json"), - this.resource.getFile()); + resource.getFile()); } @Test public void testJsonWritingWithMultipleWrite() throws Exception { + //given + Path outputFilePath = Paths.get("target", "testJsonWritingWithMultipleWrite-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(resource) + .jsonObjectMarshaller(getJsonObjectMarshaller()) + .build(); + // when - this.writer.open(this.executionContext); - this.writer.write(this.items); - this.writer.write(Arrays.asList(trade3, trade4)); - this.writer.close(); + writer.open(new ExecutionContext()); + writer.write(Arrays.asList(this.trade1, this.trade2)); + writer.write(Arrays.asList(this.trade3, this.trade4)); + writer.close(); // then assertFileEquals( new File(EXPECTED_FILE_DIRECTORY + "expected-trades-with-multiple-writes.json"), - this.resource.getFile()); + resource.getFile()); } @Test public void testJsonWritingWithPrettyPrinting() throws Exception { // given - this.writer = new JsonFileItemWriterBuilder() + Path outputFilePath = Paths.get("target", "testJsonWritingWithPrettyPrinting-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() .name("tradesItemWriter") - .resource(this.resource) + .resource(resource) .jsonObjectMarshaller(getJsonObjectMarshallerWithPrettyPrint()) .build(); // when - this.writer.open(this.executionContext); - this.writer.write(this.items); - this.writer.close(); + writer.open(new ExecutionContext()); + writer.write(Arrays.asList(this.trade1, this.trade2)); + writer.close(); // when assertFileEquals( new File(EXPECTED_FILE_DIRECTORY + getExpectedPrettyPrintedFile()), - this.resource.getFile()); + resource.getFile()); } @Test public void testJsonWritingWithEnclosingObject() throws Exception { // given - this.writer.setHeaderCallback(writer -> writer.write("{\"trades\":[")); - this.writer.setFooterCallback(writer -> writer.write(JsonFileItemWriter.DEFAULT_LINE_SEPARATOR + "]}")); + Path outputFilePath = Paths.get("target", "testJsonWritingWithEnclosingObject-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(resource) + .jsonObjectMarshaller(getJsonObjectMarshaller()) + .headerCallback(headerWriter -> headerWriter.write("{\"trades\":[")) + .footerCallback(footerWriter -> footerWriter.write(JsonFileItemWriter.DEFAULT_LINE_SEPARATOR + "]}")) + .build(); // when - this.writer.open(this.executionContext); - this.writer.write(this.items); - this.writer.close(); + writer.open(new ExecutionContext()); + writer.write(Arrays.asList(this.trade1, this.trade2)); + writer.close(); // then assertFileEquals( new File(EXPECTED_FILE_DIRECTORY + "expected-trades-with-wrapper-object.json"), - this.resource.getFile()); + resource.getFile()); } @Test public void testForcedWrite() throws Exception { // given - this.writer.setForceSync(true); + Path outputFilePath = Paths.get("target", "testForcedWrite-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(resource) + .jsonObjectMarshaller(getJsonObjectMarshaller()) + .forceSync(true) + .build(); // when - this.writer.open(this.executionContext); - this.writer.write(Collections.singletonList(this.trade1)); - this.writer.close(); + writer.open(new ExecutionContext()); + writer.write(Collections.singletonList(this.trade1)); + writer.close(); // then assertFileEquals( new File(EXPECTED_FILE_DIRECTORY + "expected-trades1.json"), - this.resource.getFile()); + resource.getFile()); } @Test public void testWriteWithDelete() throws Exception { // given - this.writer.setShouldDeleteIfExists(true); + ExecutionContext executionContext = new ExecutionContext(); + Path outputFilePath = Paths.get("target", "testWriteWithDelete-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(resource) + .jsonObjectMarshaller(getJsonObjectMarshaller()) + .shouldDeleteIfExists(true) + .build(); // when - this.writer.open(this.executionContext); - this.writer.write(Collections.singletonList(this.trade1)); - this.writer.close(); - this.writer.open(this.executionContext); - this.writer.write(Collections.singletonList(this.trade2)); - this.writer.close(); + writer.open(executionContext); + writer.write(Collections.singletonList(this.trade1)); + writer.close(); + writer.open(executionContext); + writer.write(Collections.singletonList(this.trade2)); + writer.close(); // then assertFileEquals( new File(EXPECTED_FILE_DIRECTORY + "expected-trades2.json"), - this.resource.getFile()); + resource.getFile()); } @Test public void testRestart() throws Exception { - this.writer.open(this.executionContext); + // given + ExecutionContext executionContext = new ExecutionContext(); + Path outputFilePath = Paths.get("target", "testRestart-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(resource) + .jsonObjectMarshaller(getJsonObjectMarshaller()) + .build(); + + // when + writer.open(executionContext); // write some lines - this.writer.write(Collections.singletonList(this.trade1)); + writer.write(Collections.singletonList(this.trade1)); // get restart data - this.writer.update(this.executionContext); + writer.update(executionContext); // close template - this.writer.close(); + writer.close(); // init with correct data - this.writer.open(this.executionContext); + writer.open(executionContext); // write more lines - this.writer.write(Collections.singletonList(this.trade2)); + writer.write(Collections.singletonList(this.trade2)); // get statistics - this.writer.update(this.executionContext); + writer.update(executionContext); // close template - this.writer.close(); + writer.close(); // verify what was written to the file assertFileEquals( new File(EXPECTED_FILE_DIRECTORY+ "expected-trades.json"), - this.resource.getFile()); + resource.getFile()); // 2 lines were written to the file in total - assertEquals(2, this.executionContext.getLong("tradesItemWriter.written")); + assertEquals(2, executionContext.getLong("tradesItemWriter.written")); } @Test public void testTransactionalRestart() throws Exception { - this.writer.open(this.executionContext); + // given PlatformTransactionManager transactionManager = new ResourcelessTransactionManager(); + ExecutionContext executionContext = new ExecutionContext(); + Path outputFilePath = Paths.get("target", "testTransactionalRestart-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(resource) + .jsonObjectMarshaller(getJsonObjectMarshaller()) + .build(); + + // when + writer.open(executionContext); new TransactionTemplate(transactionManager).execute((TransactionCallback) status -> { try { // write some lines - this.writer.write(Collections.singletonList(this.trade1)); + writer.write(Collections.singletonList(this.trade1)); } catch (Exception e) { throw new UnexpectedInputException("Could not write data", e); } // get restart data - this.writer.update(this.executionContext); + writer.update(executionContext); return null; }); // close template - this.writer.close(); + writer.close(); // init with correct data - this.writer.open(this.executionContext); + writer.open(executionContext); new TransactionTemplate(transactionManager).execute((TransactionCallback) status -> { try { // write more lines - this.writer.write(Collections.singletonList(this.trade2)); + writer.write(Collections.singletonList(this.trade2)); } catch (Exception e) { throw new UnexpectedInputException("Could not write data", e); } // get restart data - this.writer.update(this.executionContext); + writer.update(executionContext); return null; }); // close template - this.writer.close(); + writer.close(); // verify what was written to the file assertFileEquals( new File(EXPECTED_FILE_DIRECTORY+ "expected-trades.json"), - this.resource.getFile()); + resource.getFile()); // 2 lines were written to the file in total - assertEquals(2, this.executionContext.getLong("tradesItemWriter.written")); + assertEquals(2, executionContext.getLong("tradesItemWriter.written")); } @Test public void testItemMarshallingFailure() throws Exception { - this.writer.setJsonObjectMarshaller(item -> { - throw new IllegalArgumentException("Bad item"); - }); - this.writer.open(this.executionContext); + // given + ExecutionContext executionContext = new ExecutionContext(); + Path outputFilePath = Paths.get("target", "testItemMarshallingFailure-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(resource) + .jsonObjectMarshaller(item -> { throw new IllegalArgumentException("Bad item"); }) + .build(); + + // when + writer.open(executionContext); try { - this.writer.write(Collections.singletonList(this.trade1)); + writer.write(Collections.singletonList(this.trade1)); fail(); } catch (IllegalArgumentException iae) { assertEquals("Bad item", iae.getMessage()); } finally { - this.writer.close(); + writer.close(); } assertFileEquals( new File(EXPECTED_FILE_DIRECTORY + "empty-trades.json"), - this.resource.getFile()); + resource.getFile()); } @Test @@ -283,29 +340,35 @@ public abstract class JsonFileItemWriterFunctionalTests { * If append=true a new output file should still be created on the first run (not restart). */ public void testAppendToNotYetExistingFile() throws Exception { - Resource toBeCreated = new FileSystemResource("target/FlatFileItemWriterTests.out"); + // given + ExecutionContext executionContext = new ExecutionContext(); + Path outputFilePath = Paths.get("target", "testAppendToNotYetExistingFile-" + getMarshallerName() + ".json"); + FileSystemResource resource = new FileSystemResource(outputFilePath); + Files.deleteIfExists(outputFilePath); + JsonFileItemWriter writer = new JsonFileItemWriterBuilder() + .name("tradesItemWriter") + .resource(new FileSystemResource(outputFilePath)) + .jsonObjectMarshaller(getJsonObjectMarshaller()) + .append(true) + .build(); - File outputFile = toBeCreated.getFile(); //enable easy content reading and auto-delete the file - - assertFalse("output file does not exist yet", toBeCreated.exists()); - this.writer.setResource(toBeCreated); - this.writer.setAppendAllowed(true); - this.writer.afterPropertiesSet(); - - this.writer.open(this.executionContext); - assertTrue("output file was created", toBeCreated.exists()); - - this.writer.write(Collections.singletonList(this.trade1)); - this.writer.close(); + // when + writer.open(executionContext); + writer.write(Collections.singletonList(this.trade1)); + writer.close(); + + // then assertFileEquals( new File(EXPECTED_FILE_DIRECTORY + "expected-trades1.json"), - outputFile); - outputFile.delete(); + resource.getFile()); } private void assertFileEquals(File expected, File actual) throws Exception { - String expectedHash = DigestUtils.md5DigestAsHex(new FileInputStream(expected)); - String actualHash = DigestUtils.md5DigestAsHex(new FileInputStream(actual)); - Assert.assertEquals(expectedHash, actualHash); + JSONAssert.assertEquals(getContent(expected), getContent(actual), false); } + + private String getContent(File file) throws IOException { + return new String(Files.readAllBytes(file.toPath()), Charset.defaultCharset()); + } + }