Fix spring test to properly handle failures

Failing test cases weren't not properly handled and instead caused
test to break. Added a test case and verified it works for both JUnit
and Spock.

spring-boot-cli/samples/failures.groovy

+class FailingJUnitTests {
+	@Test
+	void passingTest() {
+		assertTrue(true)
+	}
+
+	@Test
+	void failureByAssertion() {
+		assertTrue(false)
+	}
+
+	@Test
+	void failureByException() {
+		throw new RuntimeException("This should also be handled")
+	}
+}
+
+class FailingSpockTest extends Specification {
+	def "this should pass"() {
+		expect:
+		name.size() == length
+
+		where:
+		name    | length
+		"Spock" | 5
+	}
+
+	def "this should fail on purpose as well"() {
+		when:
+		String text = "Greetings"
+
+		then:
+		//throw new RuntimeException("This should fail!")
+		true == false
+	}
+
+}
\ No newline at end of file

spring-boot-cli/src/main/groovy/testers/junit.groovy

@@ -31,39 +31,41 @@ import java.lang.reflect.Method
  */
 class JUnitTester extends AbstractTester {
 
-    @Override
-    protected Set<Class<?>> findTestableClasses(List<Class<?>> compiled) {
-        // Look for @Test methods
-        Set<Class<?>> testable = new LinkedHashSet<Class<?>>()
-        for (Class<?> clazz : compiled) {
-            for (Method method : clazz.getMethods()) {
-                for (Annotation annotation : method.getAnnotations()) {
-                    if (annotation.toString().contains("Test")) {
-                        testable.add(clazz)
-                    }
-                }
-            }
-        }
-        return testable
-    }
+	@Override
+	protected Set<Class<?>> findTestableClasses(List<Class<?>> compiled) {
+		// Look for @Test methods
+		Set<Class<?>> testable = new LinkedHashSet<Class<?>>()
+		for (Class<?> clazz : compiled) {
+			for (Method method : clazz.getMethods()) {
+				for (Annotation annotation : method.getAnnotations()) {
+					if (annotation.toString().contains("Test")) {
+						testable.add(clazz)
+					}
+				}
+			}
+		}
+		return testable
+	}
 
-    @Override
-    protected TestResults test(Class<?>[] testable) {
-        Result results = JUnitCore.runClasses(testable)
+	@Override
+	protected TestResults test(List<Class<?>> testable) {
+		return JUnitTester.runEmbeddedTests(testable)
+	}
 
-        TestResults testResults = new TestResults()
-        testResults.setFailureCount(results.getFailureCount())
-        testResults.setRunCount(results.getRunCount())
+	static TestResults runEmbeddedTests(List<Class<?>> testable) {
+		Result results = JUnitCore.runClasses(testable.toArray(new Class<?>[0]))
 
-        List<org.springframework.boot.cli.command.tester.Failure> failures =
-            new ArrayList<Failure>()
-        for (org.junit.runner.notification.Failure failure : results.getFailures()) {
-            failures.add(new Failure(failure.getDescription().toString(), failure.getTrace()))
-        }
+		TestResults testResults = new TestResults()
+		testResults.setRunCount(results.getRunCount())
 
-        testResults.setFailures(failures.toArray(new Failure[0]))
+		List<Failure> failures = new ArrayList<Failure>()
+		for (org.junit.runner.notification.Failure failure : results.getFailures()) {
+			failures.add(new Failure(failure.exception.toString(), failure.trace))
+		}
 
-        return testResults
-    }
+		testResults.setFailures(failures)
+
+		return testResults
+	}
 
 }

spring-boot-cli/src/main/groovy/testers/spock.groovy

@@ -14,49 +14,34 @@
  * limitations under the License.
  */
 
-import org.junit.runner.Result
-import org.springframework.boot.cli.command.tester.Failure
 import org.springframework.boot.cli.command.tester.TestResults
 import spock.lang.Specification
-import spock.util.EmbeddedSpecRunner
 
 /**
  * Groovy script to run Spock tests inside the {@link TestCommand}.
  * Needs to be compiled along with the actual code to work properly.
  *
+ * NOTE: SpockTester depends on JUnitTester to actually run the tests
+ *
  * @author Greg Turnquist
  */
 class SpockTester extends AbstractTester {
 
-    @Override
-    protected Set<Class<?>> findTestableClasses(List<Class<?>> compiled) {
-        // Look for classes that implement spock.lang.Specification
-        Set<Class<?>> testable = new LinkedHashSet<Class<?>>()
-        for (Class<?> clazz : compiled) {
-            if (Specification.class.isAssignableFrom(clazz)) {
-                testable.add(clazz)
-            }
-        }
-        return testable
-    }
-
-    @Override
-    protected TestResults test(Class<?>[] testable) {
-        Result results = new EmbeddedSpecRunner().runClasses(Arrays.asList(testable))
-
-        TestResults testResults = new TestResults()
-        testResults.setFailureCount(results.getFailureCount())
-        testResults.setRunCount(results.getRunCount())
-
-        List<org.springframework.boot.cli.command.tester.Failure> failures =
-            new ArrayList<Failure>()
-        for (org.junit.runner.notification.Failure failure : results.getFailures()) {
-            failures.add(new Failure(failure.getDescription().toString(), failure.getTrace()))
-        }
-
-        testResults.setFailures(failures.toArray(new Failure[0]))
-
-        return testResults
-    }
+	@Override
+	protected Set<Class<?>> findTestableClasses(List<Class<?>> compiled) {
+		// Look for classes that implement spock.lang.Specification
+		Set<Class<?>> testable = new LinkedHashSet<Class<?>>()
+		for (Class<?> clazz : compiled) {
+			if (Specification.class.isAssignableFrom(clazz)) {
+				testable.add(clazz)
+			}
+		}
+		return testable
+	}
+
+	@Override
+	protected TestResults test(List<Class<?>> testable) {
+		return JUnitTester.runEmbeddedTests(testable)
+	}
 
 }

spring-boot-cli/src/main/groovy/testers/tester.groovy

@@ -25,18 +25,18 @@ import org.springframework.boot.cli.command.tester.TestResults
  */
 public abstract class AbstractTester {
 
-    public TestResults findAndTest(List<Class<?>> compiled) throws FileNotFoundException {
-        Set<Class<?>> testable = findTestableClasses(compiled)
+	public TestResults findAndTest(List<Class<?>> compiled) throws FileNotFoundException {
+		Set<Class<?>> testable = findTestableClasses(compiled)
 
-        if (testable.size() == 0) {
-            return TestResults.NONE
-        }
+		if (testable.size() == 0) {
+			return TestResults.NONE
+		}
 
-        return test(testable.toArray(new Class<?>[0]))
-    }
+		return test(new ArrayList<Class<?>>(testable))
+	}
 
-    protected abstract Set<Class<?>> findTestableClasses(List<Class<?>> compiled)
+	protected abstract Set<Class<?>> findTestableClasses(List<Class<?>> compiled)
 
-    protected abstract TestResults test(Class<?>[] testable)
+	protected abstract TestResults test(List<Class<?>> testable)
 
 }

spring-boot-cli/src/main/java/org/springframework/boot/cli/command/TestCommand.java

@@ -88,7 +88,6 @@ public class TestCommand extends OptionParsingCommand {
 	private static class TestOptionHandler extends OptionHandler {
 
 		private final GroovyCompiler compiler;
-
 		private TestResults results;
 
 		public TestOptionHandler() {
@@ -113,11 +112,12 @@ public class TestCommand extends OptionParsingCommand {
 			 * against the composite AST.
 			 */
 
-			// Compile - Pass 1 - collect testers
+			// Compile - Pass 1 - compile source code to see what test libraries were
+			// pulled in
 			Object[] sources = this.compiler.sources(fileOptions.getFilesArray());
-			Set<File> testerFiles = compileAndCollectTesterFiles(sources);
+			List<File> testerFiles = compileAndCollectTesterFiles(sources);
 
-			// Compile - Pass 2 - with appropriate testers added in
+			// Compile - Pass 2 - add appropriate testers
 			List<File> files = new ArrayList<File>(fileOptions.getFiles());
 			files.addAll(testerFiles);
 			sources = this.compiler.sources(files.toArray(new File[files.size()]));
@@ -145,14 +145,17 @@ public class TestCommand extends OptionParsingCommand {
 				GroovyObject obj = (GroovyObject) tester.newInstance();
 				this.results.add((TestResults) obj.invokeMethod("findAndTest", compiled));
 			}
+
 			printReport(this.results);
 		}
 
-		private Set<File> compileAndCollectTesterFiles(Object[] sources)
+		private List<File> compileAndCollectTesterFiles(Object[] sources)
 				throws CompilationFailedException, IOException {
-			Set<File> testerFiles = new LinkedHashSet<File>();
-			addTesterOnClass(sources, "org.junit.Test", "junit", testerFiles);
-			addTesterOnClass(sources, "spock.lang.Specification", "spock", testerFiles);
+			Set<String> testerUnits = new LinkedHashSet<String>();
+			List<File> testerFiles = new ArrayList<File>();
+			addTesterOnClass(sources, "org.junit.Test", testerFiles, testerUnits, "junit");
+			addTesterOnClass(sources, "spock.lang.Specification", testerFiles,
+					testerUnits, "junit", "spock");
 			if (!testerFiles.isEmpty()) {
 				testerFiles.add(createTempTesterFile("tester"));
 			}
@@ -161,12 +164,16 @@ public class TestCommand extends OptionParsingCommand {
 		}
 
 		private void addTesterOnClass(Object[] sources, String className,
-				String testerName, Set<File> testerFiles) {
+				List<File> testerFiles, Set<String> testerUnits, String... testerNames) {
 			for (Object source : sources) {
 				if (source instanceof Class<?>) {
 					try {
 						((Class<?>) source).getClassLoader().loadClass(className);
-						testerFiles.add(createTempTesterFile(testerName));
+						for (String testerName : testerNames) {
+							if (testerUnits.add(testerName)) {
+								testerFiles.add(createTempTesterFile(testerName));
+							}
+						}
 						return;
 					}
 					catch (ClassNotFoundException ex) {
@@ -201,7 +208,7 @@ public class TestCommand extends OptionParsingCommand {
 			String trailer = "";
 			String trace = "";
 			for (Failure failure : results.getFailures()) {
-				trailer += failure.getDescription().toString();
+				trailer += "Failed: " + failure.getDescription().toString() + "\n";
 				trace += failure.getTrace() + "\n";
 			}
 

spring-boot-cli/src/main/java/org/springframework/boot/cli/command/tester/TestResults.java

@@ -16,7 +16,7 @@
 
 package org.springframework.boot.cli.command.tester;
 
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -42,8 +42,8 @@ public class TestResults {
 		}
 
 		@Override
-		public Failure[] getFailures() {
-			return new Failure[0];
+		public List<Failure> getFailures() {
+			return new ArrayList<Failure>();
 		}
 
 		@Override
@@ -52,23 +52,16 @@ public class TestResults {
 		}
 
 	};
-
-	private int runCount;
-
-	private int failureCount;
-
-	private Failure[] failures = new Failure[0];
+	private int runCount = 0;
+	private List<Failure> failures = new ArrayList<Failure>();
 
 	public void add(TestResults results) {
 		this.runCount += results.getRunCount();
-		this.failureCount += results.getFailureCount();
-		List<Failure> failures = Arrays.asList(this.failures);
-		failures.addAll(Arrays.asList(results.getFailures()));
-		this.failures = failures.toArray(new Failure[] {});
+		this.failures.addAll(results.getFailures());
 	}
 
 	public boolean wasSuccessful() {
-		return this.failureCount == 0;
+		return this.failures.size() == 0;
 	}
 
 	public int getRunCount() {
@@ -80,18 +73,14 @@ public class TestResults {
 	}
 
 	public int getFailureCount() {
-		return this.failureCount;
-	}
-
-	public void setFailureCount(int failureCount) {
-		this.failureCount = failureCount;
+		return this.failures.size();
 	}
 
-	public Failure[] getFailures() {
+	public List<Failure> getFailures() {
 		return this.failures;
 	}
 
-	public void setFailures(Failure[] failures) {
+	public void setFailures(List<Failure> failures) {
 		this.failures = failures;
 	}
 

spring-boot-cli/src/main/java/org/springframework/boot/cli/compiler/autoconfigure/SpockCompilerAutoConfiguration.java

@@ -38,12 +38,16 @@ public class SpockCompilerAutoConfiguration extends CompilerAutoConfiguration {
 	@Override
 	public void applyDependencies(DependencyCustomizer dependencies)
 			throws CompilationFailedException {
-		dependencies.add("spock-core");
+		dependencies.add("spock-core").add("junit").add("spring-test")
+				.add("hamcrest-library");
 	}
 
 	@Override
 	public void applyImports(ImportCustomizer imports) throws CompilationFailedException {
-		imports.addStarImports("spock.lang");
+		imports.addStarImports("spock.lang").addStarImports("org.junit")
+				.addStaticStars("org.junit.Assert")
+				.addStaticStars("org.hamcrest.MatcherAssert")
+				.addStaticStars("org.hamcrest.Matchers");
 	}
 
 }

spring-boot-cli/src/test/java/org/springframework/boot/cli/TestCommandIntegrationTests.java

@@ -25,6 +25,7 @@ import org.springframework.boot.cli.command.TestCommand;
 import org.springframework.boot.cli.command.tester.TestResults;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -37,12 +38,11 @@ public class TestCommandIntegrationTests {
 	@BeforeClass
 	public static void cleanGrapes() throws Exception {
 		GrapesCleaner.cleanIfNecessary();
-		// System.setProperty("ivy.message.logger.level", "3");
 	}
 
 	@Before
 	public void setup() throws Exception {
-		System.setProperty("disableSpringSnapshotRepos", "true");
+		System.setProperty("disableSpringSnapshotRepos", "false");
 		new CleanCommand().run("org.springframework");
 	}
 
@@ -77,9 +77,9 @@ public class TestCommandIntegrationTests {
 			TestCommand command = new TestCommand();
 			command.run("samples/nothing.groovy");
 		}
-		catch (RuntimeException e) {
-			assertEquals("Can't find samples/nothing.groovy", e.getMessage());
-			throw e;
+		catch (RuntimeException ex) {
+			assertEquals("Can't find samples/nothing.groovy", ex.getMessage());
+			throw ex;
 		}
 	}
 
@@ -123,4 +123,14 @@ public class TestCommandIntegrationTests {
 		assertTrue(results.wasSuccessful());
 	}
 
+	@Test
+	public void verifyFailures() throws Exception {
+		TestCommand command = new TestCommand();
+		command.run("samples/failures.groovy");
+		TestResults results = command.getResults();
+		assertEquals(5, results.getRunCount());
+		assertEquals(3, results.getFailureCount());
+		assertFalse(results.wasSuccessful());
+	}
+
 }