Preserve UTF-8 content when masking links in JSON payloads

Closes gh-705

spring-restdocs-core/src/main/java/org/springframework/restdocs/operation/preprocess/LinkMaskingContentModifier.java

@@ -16,6 +16,7 @@
 
 package org.springframework.restdocs.operation.preprocess;
 
+import java.nio.charset.StandardCharsets;
 import java.util.regex.Pattern;
 
 import org.springframework.http.MediaType;
@@ -38,7 +39,7 @@ class LinkMaskingContentModifier implements ContentModifier {
 	}
 
 	LinkMaskingContentModifier(String mask) {
-		this.contentModifier = new PatternReplacingContentModifier(LINK_HREF, mask);
+		this.contentModifier = new PatternReplacingContentModifier(LINK_HREF, mask, StandardCharsets.UTF_8);
 	}
 
 	@Override

spring-restdocs-core/src/main/java/org/springframework/restdocs/operation/preprocess/PatternReplacingContentModifier.java

@@ -16,6 +16,7 @@
 
 package org.springframework.restdocs.operation.preprocess;
 
+import java.nio.charset.Charset;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -34,26 +35,40 @@ class PatternReplacingContentModifier implements ContentModifier {
 
 	private final String replacement;
 
+	private final Charset fallbackCharset;
+
 	/**
 	 * Creates a new {@link PatternReplacingContentModifier} that will replace occurrences
-	 * of the given {@code pattern} with the given {@code replacement}.
+	 * of the given {@code pattern} with the given {@code replacement}. The content is
+	 * handled using the charset from its content type. When no content type is specified
+	 * the JVM's {@link Charset#defaultCharset() default charset is used}.
 	 * @param pattern the pattern
 	 * @param replacement the replacement
 	 */
 	PatternReplacingContentModifier(Pattern pattern, String replacement) {
+		this(pattern, replacement, Charset.defaultCharset());
+	}
+
+	/**
+	 * Creates a new {@link PatternReplacingContentModifier} that will replace occurrences
+	 * of the given {@code pattern} with the given {@code replacement}. The content is
+	 * handled using the charset from its content type. When no content type is specified
+	 * the given {@code fallbackCharset} is used.
+	 * @param pattern the pattern
+	 * @param replacement the replacement
+	 * @param fallbackCharset the charset to use as a fallback
+	 */
+	PatternReplacingContentModifier(Pattern pattern, String replacement, Charset fallbackCharset) {
 		this.pattern = pattern;
 		this.replacement = replacement;
+		this.fallbackCharset = fallbackCharset;
 	}
 
 	@Override
 	public byte[] modifyContent(byte[] content, MediaType contentType) {
-		String original;
-		if (contentType != null && contentType.getCharset() != null) {
-			original = new String(content, contentType.getCharset());
-		}
-		else {
-			original = new String(content);
-		}
+		Charset charset = (contentType != null && contentType.getCharset() != null) ? contentType.getCharset()
+				: this.fallbackCharset;
+		String original = new String(content, charset);
 		Matcher matcher = this.pattern.matcher(original);
 		StringBuilder builder = new StringBuilder();
 		int previous = 0;
@@ -73,7 +88,7 @@ class PatternReplacingContentModifier implements ContentModifier {
 		if (previous < original.length()) {
 			builder.append(original.substring(previous));
 		}
-		return builder.toString().getBytes();
+		return builder.toString().getBytes(charset);
 	}
 
 }

spring-restdocs-core/src/test/java/org/springframework/restdocs/operation/preprocess/LinkMaskingContentModifierTests.java

@@ -77,6 +77,14 @@ public class LinkMaskingContentModifierTests {
 						.isEqualTo(formattedAtomPayloadWithLinks(new Link("a", "custom"), new Link("b", "custom")));
 	}
 
+	@Test
+	public void maskCanUseUtf8Characters() throws Exception {
+		String ellipsis = "\u2026";
+		assertThat(
+				new LinkMaskingContentModifier(ellipsis).modifyContent(formattedHalPayloadWithLinks(this.links), null))
+						.isEqualTo(formattedHalPayloadWithLinks(new Link("a", ellipsis), new Link("b", ellipsis)));
+	}
+
 	private byte[] atomPayloadWithLinks(Link... links) throws JsonProcessingException {
 		return new ObjectMapper().writeValueAsBytes(createAtomPayload(links));
 	}

spring-restdocs-core/src/test/java/org/springframework/restdocs/operation/preprocess/PatternReplacingContentModifierTests.java

@@ -17,6 +17,7 @@
 package org.springframework.restdocs.operation.preprocess;
 
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.regex.Pattern;
 
 import org.junit.Test;
@@ -52,7 +53,7 @@ public class PatternReplacingContentModifierTests {
 	}
 
 	@Test
-	public void encodingIsPreserved() {
+	public void encodingIsPreservedUsingCharsetFromContentType() {
 		String japaneseContent = "\u30b3\u30f3\u30c6\u30f3\u30c4";
 		Pattern pattern = Pattern.compile("[0-9]+");
 		PatternReplacingContentModifier contentModifier = new PatternReplacingContentModifier(pattern, "<<number>>");
@@ -61,4 +62,14 @@ public class PatternReplacingContentModifierTests {
 						.isEqualTo((japaneseContent + " <<number>>").getBytes());
 	}
 
+	@Test
+	public void encodingIsPreservedUsingFallbackCharset() {
+		String japaneseContent = "\u30b3\u30f3\u30c6\u30f3\u30c4";
+		Pattern pattern = Pattern.compile("[0-9]+");
+		PatternReplacingContentModifier contentModifier = new PatternReplacingContentModifier(pattern, "<<number>>",
+				StandardCharsets.UTF_8);
+		assertThat(contentModifier.modifyContent((japaneseContent + " 123").getBytes(), new MediaType("text", "plain")))
+				.isEqualTo((japaneseContent + " <<number>>").getBytes());
+	}
+
 }