Require occasionally null or absent field in array to be marked optional

Closes gh-402

spring-restdocs-core/src/main/java/org/springframework/restdocs/payload/JsonFieldProcessor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2016 the original author or authors.
+ * Copyright 2014-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@ import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * A {@code JsonFieldProcessor} processes a payload's fields, allowing them to be
@@ -32,17 +31,10 @@ import java.util.concurrent.atomic.AtomicReference;
  */
 final class JsonFieldProcessor {
 
-	boolean hasField(JsonFieldPath fieldPath, Object payload) {
-		final AtomicReference<Boolean> hasField = new AtomicReference<>(false);
-		traverse(new ProcessingContext(payload, fieldPath), new MatchCallback() {
-
-			@Override
-			public void foundMatch(Match match) {
-				hasField.set(true);
-			}
-
-		});
-		return hasField.get();
+	boolean hasField(final JsonFieldPath fieldPath, Object payload) {
+		HasFieldMatchCallback callback = new HasFieldMatchCallback();
+		traverse(new ProcessingContext(payload, fieldPath), callback);
+		return callback.fieldFound();
 	}
 
 	Object extract(JsonFieldPath path, Object payload) {
@@ -54,6 +46,11 @@ final class JsonFieldProcessor {
 				matches.add(match.getValue());
 			}
 
+			@Override
+			public void absent() {
+
+			}
+
 		});
 		if (matches.isEmpty()) {
 			throw new FieldDoesNotExistException(path);
@@ -74,6 +71,11 @@ final class JsonFieldProcessor {
 				match.remove();
 			}
 
+			@Override
+			public void absent() {
+
+			}
+
 		});
 	}
 
@@ -85,6 +87,11 @@ final class JsonFieldProcessor {
 				match.removeSubsection();
 			}
 
+			@Override
+			public void absent() {
+
+			}
+
 		});
 	}
 
@@ -128,8 +135,8 @@ final class JsonFieldProcessor {
 	private void handleMapPayload(ProcessingContext context,
 			MatchCallback matchCallback) {
 		Map<?, ?> map = context.getPayload();
-		Object item = map.get(context.getSegment());
-		if (item != null || map.containsKey(context.getSegment())) {
+		if (map.containsKey(context.getSegment())) {
+			Object item = map.get(context.getSegment());
 			MapMatch mapMatch = new MapMatch(item, map, context.getSegment(),
 					context.getParentMatch());
 			if (context.isLeaf()) {
@@ -142,6 +149,47 @@ final class JsonFieldProcessor {
 		else if ("*".equals(context.getSegment())) {
 			handleCollectionPayload(map.values(), matchCallback, context);
 		}
+		else {
+			matchCallback.absent();
+		}
+	}
+
+	/**
+	 * {@link MatchCallback} use to determine whether a payload has a particular field.
+	 */
+	private static final class HasFieldMatchCallback implements MatchCallback {
+
+		private MatchType matchType = MatchType.NONE;
+
+		@Override
+		public void foundMatch(Match match) {
+			this.matchType = this.matchType.combinedWith(
+					match.getValue() == null ? MatchType.NULL : MatchType.NON_NULL);
+		}
+
+		@Override
+		public void absent() {
+			this.matchType = this.matchType.combinedWith(MatchType.ABSENT);
+		}
+
+		boolean fieldFound() {
+			return this.matchType == MatchType.NON_NULL
+					|| this.matchType == MatchType.NULL;
+		}
+
+		private static enum MatchType {
+
+			ABSENT, MIXED, NONE, NULL, NON_NULL;
+
+			MatchType combinedWith(MatchType matchType) {
+				if (this == NONE || this == matchType) {
+					return matchType;
+				}
+				return MIXED;
+			}
+
+		}
+
 	}
 
 	private static final class MapMatch implements Match {
@@ -265,6 +313,8 @@ final class JsonFieldProcessor {
 
 		void foundMatch(Match match);
 
+		void absent();
+
 	}
 
 	private interface Match {

spring-restdocs-core/src/test/java/org/springframework/restdocs/payload/JsonFieldProcessorTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2016 the original author or authors.
+ * Copyright 2014-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.Test;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.hasEntry;
@@ -105,6 +106,31 @@ public class JsonFieldProcessorTests {
 				equalTo((Object) Arrays.asList("bravo", "bravo")));
 	}
 
+	@Test
+	public void extractOccasionallyAbsentFieldFromItemsInArray() {
+		Map<String, Object> payload = new HashMap<>();
+		Map<String, Object> entry = new HashMap<>();
+		entry.put("b", "bravo");
+		List<Map<String, Object>> alpha = Arrays.asList(entry,
+				new HashMap<String, Object>());
+		payload.put("a", alpha);
+		assertThat(this.fieldProcessor.extract(JsonFieldPath.compile("a[].b"), payload),
+				equalTo((Object) Arrays.asList("bravo")));
+	}
+
+	@Test
+	public void extractOccasionallyNullFieldFromItemsInArray() {
+		Map<String, Object> payload = new HashMap<>();
+		Map<String, Object> nonNullField = new HashMap<>();
+		nonNullField.put("b", "bravo");
+		Map<String, Object> nullField = new HashMap<>();
+		nullField.put("b", null);
+		List<Map<String, Object>> alpha = Arrays.asList(nonNullField, nullField);
+		payload.put("a", alpha);
+		assertThat(this.fieldProcessor.extract(JsonFieldPath.compile("a[].b"), payload),
+				equalTo((Object) Arrays.asList("bravo", null)));
+	}
+
 	@Test
 	public void extractNestedArray() {
 		Map<String, Object> payload = new HashMap<>();
@@ -406,6 +432,82 @@ public class JsonFieldProcessorTests {
 		assertThat(payload, hasEntry("c", (Object) "charlie"));
 	}
 
+	@Test
+	public void hasFieldIsTrueForNonNullFieldInMap() throws Exception {
+		Map<String, Object> payload = new HashMap<>();
+		payload.put("a", "alpha");
+		assertThat(this.fieldProcessor.hasField(JsonFieldPath.compile("a"), payload),
+				is(true));
+	}
+
+	@Test
+	public void hasFieldIsTrueForNullFieldInMap() throws Exception {
+		Map<String, Object> payload = new HashMap<>();
+		payload.put("a", null);
+		assertThat(this.fieldProcessor.hasField(JsonFieldPath.compile("a"), payload),
+				is(true));
+	}
+
+	@Test
+	public void hasFieldIsFalseForAbsentFieldInMap() throws Exception {
+		Map<String, Object> payload = new HashMap<>();
+		payload.put("a", null);
+		assertThat(this.fieldProcessor.hasField(JsonFieldPath.compile("b"), payload),
+				is(false));
+	}
+
+	@Test
+	public void hasFieldIsTrueForNeverNullFieldBeneathArray() throws Exception {
+		Map<String, Object> payload = new HashMap<>();
+		Map<String, Object> nested = new HashMap<>();
+		nested.put("b", "bravo");
+		payload.put("a", Arrays.asList(nested, nested, nested));
+		assertThat(this.fieldProcessor.hasField(JsonFieldPath.compile("a.[].b"), payload),
+				is(true));
+	}
+
+	@Test
+	public void hasFieldIsTrueForAlwaysNullFieldBeneathArray() throws Exception {
+		Map<String, Object> payload = new HashMap<>();
+		Map<String, Object> nested = new HashMap<>();
+		nested.put("b", null);
+		payload.put("a", Arrays.asList(nested, nested, nested));
+		assertThat(this.fieldProcessor.hasField(JsonFieldPath.compile("a.[].b"), payload),
+				is(true));
+	}
+
+	@Test
+	public void hasFieldIsFalseForAlwaysAbsentFieldBeneathArray() throws Exception {
+		Map<String, Object> payload = new HashMap<>();
+		Map<String, Object> nested = new HashMap<>();
+		nested.put("b", "bravo");
+		payload.put("a", Arrays.asList(nested, nested, nested));
+		assertThat(this.fieldProcessor.hasField(JsonFieldPath.compile("a.[].c"), payload),
+				is(false));
+	}
+
+	@Test
+	public void hasFieldIsFalseForOccasionallyAbsentFieldBeneathArray() throws Exception {
+		Map<String, Object> payload = new HashMap<>();
+		Map<String, Object> nested = new HashMap<>();
+		nested.put("b", "bravo");
+		payload.put("a", Arrays.asList(nested, new HashMap<>(), nested));
+		assertThat(this.fieldProcessor.hasField(JsonFieldPath.compile("a.[].b"), payload),
+				is(false));
+	}
+
+	@Test
+	public void hasFieldIsFalseForOccasionallyNullFieldBeneathArray() throws Exception {
+		Map<String, Object> payload = new HashMap<>();
+		Map<String, Object> fieldPresent = new HashMap<>();
+		fieldPresent.put("b", "bravo");
+		Map<String, Object> fieldNull = new HashMap<>();
+		fieldNull.put("b", null);
+		payload.put("a", Arrays.asList(fieldPresent, fieldPresent, fieldNull));
+		assertThat(this.fieldProcessor.hasField(JsonFieldPath.compile("a.[].b"), payload),
+				is(false));
+	}
+
 	private Map<String, String> createEntry(String... pairs) {
 		Map<String, String> entry = new HashMap<>();
 		for (String pair : pairs) {

spring-restdocs-core/src/test/java/org/springframework/restdocs/payload/RequestFieldsSnippetFailureTests.java

@@ -180,8 +180,8 @@ public class RequestFieldsSnippetFailureTests {
 	public void undocumentedXmlRequestFieldAndMissingXmlRequestField()
 			throws IOException {
 		this.thrown.expect(SnippetException.class);
-		this.thrown.expectMessage(startsWith(
-				"The following parts of the payload were not" + " documented:"));
+		this.thrown.expectMessage(
+				startsWith("The following parts of the payload were not documented:"));
 		this.thrown
 				.expectMessage(endsWith("Fields with the following paths were not found"
 						+ " in the payload: [a/b]"));
@@ -204,4 +204,36 @@ public class RequestFieldsSnippetFailureTests {
 						.build());
 	}
 
+	@Test
+	public void nonOptionalFieldBeneathArrayThatIsSometimesNull() throws IOException {
+		this.thrown.expect(SnippetException.class);
+		this.thrown.expectMessage(startsWith(
+				"Fields with the following paths were not found in the payload: "
+						+ "[a[].b]"));
+		new RequestFieldsSnippet(Arrays.asList(
+				fieldWithPath("a[].b").description("one").type(JsonFieldType.NUMBER),
+				fieldWithPath("a[].c").description("two").type(JsonFieldType.NUMBER)))
+						.document(this.operationBuilder.request("http://localhost")
+								.content("{\"a\":[{\"b\": 1,\"c\": 2}, "
+										+ "{\"b\": null, \"c\": 2},"
+										+ " {\"b\": 1,\"c\": 2}]}")
+								.build());
+	}
+
+	@Test
+	public void nonOptionalFieldBeneathArrayThatIsSometimesAbsent() throws IOException {
+		this.thrown.expect(SnippetException.class);
+		this.thrown.expectMessage(startsWith(
+				"Fields with the following paths were not found in the payload: "
+						+ "[a[].b]"));
+		new RequestFieldsSnippet(Arrays.asList(
+				fieldWithPath("a[].b").description("one").type(JsonFieldType.NUMBER),
+				fieldWithPath("a[].c").description("two").type(JsonFieldType.NUMBER)))
+						.document(
+								this.operationBuilder.request("http://localhost")
+										.content("{\"a\":[{\"b\": 1,\"c\": 2}, "
+												+ "{\"c\": 2}, {\"b\": 1,\"c\": 2}]}")
+								.build());
+	}
+
 }

spring-restdocs-core/src/test/java/org/springframework/restdocs/payload/RequestFieldsSnippetTests.java

@@ -66,6 +66,17 @@ public class RequestFieldsSnippetTests extends AbstractSnippetTests {
 								.build());
 	}
 
+	@Test
+	public void mapRequestWithNullField() throws IOException {
+		this.snippets.expectRequestFields()
+				.withContents(tableWithHeader("Path", "Type", "Description").row("`a.b`",
+						"`Null`", "one"));
+
+		new RequestFieldsSnippet(Arrays.asList(fieldWithPath("a.b").description("one")))
+				.document(this.operationBuilder.request("http://localhost")
+						.content("{\"a\": {\"b\": null}}").build());
+	}
+
 	@Test
 	public void entireSubsectionsCanBeDocumented() throws IOException {
 		this.snippets.expectRequestFields()
@@ -105,8 +116,20 @@ public class RequestFieldsSnippetTests extends AbstractSnippetTests {
 				fieldWithPath("[]a.c").description("three"),
 				fieldWithPath("[]a").description("four")))
 						.document(this.operationBuilder.request("http://localhost")
-								.content(
-										"[{\"a\": {\"b\": 5}},{\"a\": {\"c\": \"charlie\"}}]")
+								.content("[{\"a\": {\"b\": 5, \"c\":\"charlie\"}},"
+										+ "{\"a\": {\"b\": 4, \"c\":\"chalk\"}}]")
+						.build());
+	}
+
+	@Test
+	public void arrayRequestWithAlwaysNullField() throws IOException {
+		this.snippets.expectRequestFields()
+				.withContents(tableWithHeader("Path", "Type", "Description")
+						.row("`[]a.b`", "`Null`", "one"));
+
+		new RequestFieldsSnippet(Arrays.asList(fieldWithPath("[]a.b").description("one")))
+				.document(this.operationBuilder.request("http://localhost")
+						.content("[{\"a\": {\"b\": null}}," + "{\"a\": {\"b\": null}}]")
 						.build());
 	}
 
@@ -388,7 +411,7 @@ public class RequestFieldsSnippetTests extends AbstractSnippetTests {
 				.withContents(tableWithHeader("Path", "Type", "Description")
 						.row("`assets[].name`", "`String`", "one"));
 		new RequestFieldsSnippet(Arrays.asList(fieldWithPath("assets[].name")
-				.description("one").type(JsonFieldType.STRING)))
+				.description("one").type(JsonFieldType.STRING).optional()))
 						.document(this.operationBuilder.request("http://localhost")
 								.content("{\"assets\": [" + "{\"name\": \"sample1\"}, "
 										+ "{\"name\": null}, "
@@ -396,6 +419,23 @@ public class RequestFieldsSnippetTests extends AbstractSnippetTests {
 								.build());
 	}
 
+	@Test
+	public void optionalFieldBeneathArrayThatIsSometimesAbsent() throws IOException {
+		this.snippets.expectRequestFields()
+				.withContents(tableWithHeader("Path", "Type", "Description")
+						.row("`a[].b`", "`Number`", "one")
+						.row("`a[].c`", "`Number`", "two"));
+		new RequestFieldsSnippet(Arrays.asList(
+				fieldWithPath("a[].b").description("one").type(JsonFieldType.NUMBER)
+						.optional(),
+				fieldWithPath("a[].c").description("two").type(JsonFieldType.NUMBER)))
+						.document(
+								this.operationBuilder.request("http://localhost")
+										.content("{\"a\":[{\"b\": 1,\"c\": 2}, "
+												+ "{\"c\": 2}, {\"b\": 1,\"c\": 2}]}")
+										.build());
+	}
+
 	private String escapeIfNecessary(String input) {
 		if (this.templateFormat.equals(TemplateFormats.markdown())) {
 			return input;

spring-restdocs-core/src/test/java/org/springframework/restdocs/payload/ResponseFieldsSnippetFailureTests.java

@@ -173,4 +173,36 @@ public class ResponseFieldsSnippetFailureTests {
 						.build());
 	}
 
+	@Test
+	public void nonOptionalFieldBeneathArrayThatIsSometimesNull() throws IOException {
+		this.thrown.expect(SnippetException.class);
+		this.thrown.expectMessage(startsWith(
+				"Fields with the following paths were not found in the payload: "
+						+ "[a[].b]"));
+		new ResponseFieldsSnippet(Arrays.asList(
+				fieldWithPath("a[].b").description("one").type(JsonFieldType.NUMBER),
+				fieldWithPath("a[].c").description("two").type(JsonFieldType.NUMBER)))
+						.document(this.operationBuilder.response()
+								.content("{\"a\":[{\"b\": 1,\"c\": 2}, "
+										+ "{\"b\": null, \"c\": 2},"
+										+ " {\"b\": 1,\"c\": 2}]}")
+								.build());
+	}
+
+	@Test
+	public void nonOptionalFieldBeneathArrayThatIsSometimesAbsent() throws IOException {
+		this.thrown.expect(SnippetException.class);
+		this.thrown.expectMessage(startsWith(
+				"Fields with the following paths were not found in the payload: "
+						+ "[a[].b]"));
+		new ResponseFieldsSnippet(Arrays.asList(
+				fieldWithPath("a[].b").description("one").type(JsonFieldType.NUMBER),
+				fieldWithPath("a[].c").description("two").type(JsonFieldType.NUMBER)))
+						.document(
+								this.operationBuilder.response()
+										.content("{\"a\":[{\"b\": 1,\"c\": 2}, "
+												+ "{\"c\": 2}, {\"b\": 1,\"c\": 2}]}")
+								.build());
+	}
+
 }

spring-restdocs-core/src/test/java/org/springframework/restdocs/payload/ResponseFieldsSnippetTests.java

@@ -72,6 +72,17 @@ public class ResponseFieldsSnippetTests extends AbstractSnippetTests {
 								.build());
 	}
 
+	@Test
+	public void mapResponseWithNullField() throws IOException {
+		this.snippets.expectResponseFields()
+				.withContents(tableWithHeader("Path", "Type", "Description").row("`a.b`",
+						"`Null`", "one"));
+
+		new ResponseFieldsSnippet(Arrays.asList(fieldWithPath("a.b").description("one")))
+				.document(this.operationBuilder.response()
+						.content("{\"a\": {\"b\": null}}").build());
+	}
+
 	@Test
 	public void subsectionOfMapResponse() throws IOException {
 		this.snippets.expect("response-fields-beneath-a")
@@ -95,11 +106,24 @@ public class ResponseFieldsSnippetTests extends AbstractSnippetTests {
 				fieldWithPath("[]a.c").description("two"),
 				fieldWithPath("[]a").description("three")))
 						.document(this.operationBuilder.response()
-								.content(
-										"[{\"a\": {\"b\": 5}},{\"a\": {\"c\": \"charlie\"}}]")
+								.content("[{\"a\": {\"b\": 5, \"c\":\"charlie\"}},"
+										+ "{\"a\": {\"b\": 4, \"c\":\"chalk\"}}]")
 								.build());
 	}
 
+	@Test
+	public void arrayResponseWithAlwaysNullField() throws IOException {
+		this.snippets.expectResponseFields()
+				.withContents(tableWithHeader("Path", "Type", "Description")
+						.row("`[]a.b`", "`Null`", "one"));
+
+		new ResponseFieldsSnippet(
+				Arrays.asList(fieldWithPath("[]a.b").description("one")))
+						.document(this.operationBuilder.response().content(
+								"[{\"a\": {\"b\": null}}," + "{\"a\": {\"b\": null}}]")
+						.build());
+	}
+
 	@Test
 	public void arrayResponse() throws IOException {
 		this.snippets.expectResponseFields()
@@ -399,7 +423,7 @@ public class ResponseFieldsSnippetTests extends AbstractSnippetTests {
 				.withContents(tableWithHeader("Path", "Type", "Description")
 						.row("`assets[].name`", "`String`", "one"));
 		new ResponseFieldsSnippet(Arrays.asList(fieldWithPath("assets[].name")
-				.description("one").type(JsonFieldType.STRING)))
+				.description("one").type(JsonFieldType.STRING).optional()))
 						.document(this.operationBuilder.response()
 								.content("{\"assets\": [" + "{\"name\": \"sample1\"}, "
 										+ "{\"name\": null}, "
@@ -407,6 +431,23 @@ public class ResponseFieldsSnippetTests extends AbstractSnippetTests {
 								.build());
 	}
 
+	@Test
+	public void optionalFieldBeneathArrayThatIsSometimesAbsent() throws IOException {
+		this.snippets.expectResponseFields()
+				.withContents(tableWithHeader("Path", "Type", "Description")
+						.row("`a[].b`", "`Number`", "one")
+						.row("`a[].c`", "`Number`", "two"));
+		new ResponseFieldsSnippet(Arrays.asList(
+				fieldWithPath("a[].b").description("one").type(JsonFieldType.NUMBER)
+						.optional(),
+				fieldWithPath("a[].c").description("two").type(JsonFieldType.NUMBER)))
+						.document(
+								this.operationBuilder.response()
+										.content("{\"a\":[{\"b\": 1,\"c\": 2}, "
+												+ "{\"c\": 2}, {\"b\": 1,\"c\": 2}]}")
+										.build());
+	}
+
 	private String escapeIfNecessary(String input) {
 		if (this.templateFormat.equals(TemplateFormats.markdown())) {
 			return input;