Spring/spring-framework
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support top-level scalar values in Jackson2Tokenizer

Browse Source 
Issue: SPR-16166
This commit is contained in:
Arjen Poutsma
2017-12-14 16:44:19 +01:00
parent 51aa16f85f
commit 8e253a316d
3 changed files with 118 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java
View File

@@ -17,12 +17,10 @@
package org.springframework.http.codec.json;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.lang.annotation.Annotation;
import java.util.Map;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -89,15 +87,9 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
}
private Flux<TokenBuffer> tokenize(Publisher<DataBuffer> input, boolean tokenizeArrayElements) {
try {
JsonFactory factory = getObjectMapper().getFactory();
JsonParser parser = factory.createNonBlockingByteArrayParser();
Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, tokenizeArrayElements);
return Flux.from(input).flatMap(tokenizer).doFinally(t -> tokenizer.endOfInput());
}
catch (IOException ex) {
return Flux.error(new UncheckedIOException(ex));
}
Flux<DataBuffer> inputFlux = Flux.from(input);
JsonFactory factory = getObjectMapper().getFactory();
return Jackson2Tokenizer.tokenize(inputFlux, factory, tokenizeArrayElements);
}
private Flux<Object> decodeInternal(Flux<TokenBuffer> tokens, ResolvableType elementType,

107
spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java
View File

@@ -21,11 +21,13 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.core.async.ByteArrayFeeder;
import com.fasterxml.jackson.databind.util.TokenBuffer;
import org.jetbrains.annotations.NotNull;
import reactor.core.publisher.Flux;
import org.springframework.core.codec.DecodingException;
@@ -41,7 +43,7 @@ import org.springframework.util.Assert;
* @author Arjen Poutsma
* @since 5.0
*/
class Jackson2Tokenizer implements Function<DataBuffer, Flux<TokenBuffer>> {
class Jackson2Tokenizer {
private final JsonParser parser;
@@ -57,15 +59,7 @@ class Jackson2Tokenizer implements Function<DataBuffer, Flux<TokenBuffer>> {
private final ByteArrayFeeder inputFeeder;
/**
* Create a new instance of the {@code Jackson2Tokenizer}.
* @param parser the non-blocking parser, obtained via
* {@link com.fasterxml.jackson.core.JsonFactory#createNonBlockingByteArrayParser}
* @param tokenizeArrayElements if {@code true} and the "top level" JSON
* object is an array, each element is returned individually, immediately
* after it is received.
*/
public Jackson2Tokenizer(JsonParser parser, boolean tokenizeArrayElements) {
private Jackson2Tokenizer(JsonParser parser, boolean tokenizeArrayElements) {
Assert.notNull(parser, "'parser' must not be null");
this.parser = parser;
@@ -74,42 +68,78 @@ class Jackson2Tokenizer implements Function<DataBuffer, Flux<TokenBuffer>> {
this.inputFeeder = (ByteArrayFeeder) this.parser.getNonBlockingInputFeeder();
}
/**
* Tokenize the given {@link DataBuffer} flux into a {@link TokenBuffer} flux, given the
* parameters.
* @param dataBuffers the source data buffers
* @param jsonFactory the factory to use
* @param tokenizeArrayElements if {@code true} and the "top level" JSON
* object is an array, each element is returned individually, immediately
* after it is received.
* @return the result token buffers
*/
public static Flux<TokenBuffer> tokenize(Flux<DataBuffer> dataBuffers, JsonFactory jsonFactory,
boolean tokenizeArrayElements) {
try {
Jackson2Tokenizer tokenizer =
new Jackson2Tokenizer(jsonFactory.createNonBlockingByteArrayParser(),
tokenizeArrayElements);
return dataBuffers.flatMap(tokenizer::tokenize, Flux::error, tokenizer::endOfInput);
}
catch (IOException ex) {
return Flux.error(ex);
}
}
@Override
public Flux<TokenBuffer> apply(DataBuffer dataBuffer) {
private Flux<TokenBuffer> tokenize(DataBuffer dataBuffer) {
byte[] bytes = new byte[dataBuffer.readableByteCount()];
dataBuffer.read(bytes);
DataBufferUtils.release(dataBuffer);
try {
this.inputFeeder.feedInput(bytes, 0, bytes.length);
List<TokenBuffer> result = new ArrayList<>();
while (true) {
JsonToken token = this.parser.nextToken();
if (token == JsonToken.NOT_AVAILABLE) {
break;
}
updateDepth(token);
if (!this.tokenizeArrayElements) {
processTokenNormal(token, result);
}
else {
processTokenArray(token, result);
}
}
return Flux.fromIterable(result);
return parseTokenBufferFlux();
}
catch (JsonProcessingException ex) {
return Flux.error(new DecodingException(
"JSON decoding error: " + ex.getOriginalMessage(), ex));
}
catch (Exception ex) {
catch (IOException ex) {
return Flux.error(ex);
}
}
private Flux<TokenBuffer> endOfInput() {
this.inputFeeder.endOfInput();
try {
return parseTokenBufferFlux();
}
catch (IOException ex) {
return Flux.error(ex);
}
}
@NotNull
private Flux<TokenBuffer> parseTokenBufferFlux() throws IOException {
List<TokenBuffer> result = new ArrayList<>();
while (true) {
JsonToken token = this.parser.nextToken();
if (token == null || token == JsonToken.NOT_AVAILABLE) {
break;
}
updateDepth(token);
if (!this.tokenizeArrayElements) {
processTokenNormal(token, result);
}
else {
processTokenArray(token, result);
}
}
return Flux.fromIterable(result);
}
private void updateDepth(JsonToken token) {
switch (token) {
case START_OBJECT:
@@ -130,11 +160,10 @@ class Jackson2Tokenizer implements Function<DataBuffer, Flux<TokenBuffer>> {
private void processTokenNormal(JsonToken token, List<TokenBuffer> result) throws IOException {
this.tokenBuffer.copyCurrentEvent(this.parser);
if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) {
if (this.objectDepth == 0 && this.arrayDepth == 0) {
result.add(this.tokenBuffer);
this.tokenBuffer = new TokenBuffer(this.parser);
}
if ((token.isStructEnd() || token.isScalarValue()) &&
this.objectDepth == 0 && this.arrayDepth == 0) {
result.add(this.tokenBuffer);
this.tokenBuffer = new TokenBuffer(this.parser);
}
}
@@ -144,8 +173,8 @@ class Jackson2Tokenizer implements Function<DataBuffer, Flux<TokenBuffer>> {
this.tokenBuffer.copyCurrentEvent(this.parser);
}
if (token == JsonToken.END_OBJECT && this.objectDepth == 0 &&
(this.arrayDepth == 1 || this.arrayDepth == 0)) {
if ((token == JsonToken.END_OBJECT && this.objectDepth == 0 && (this.arrayDepth == 1 || this.arrayDepth == 0)) ||
(token.isScalarValue()) && this.objectDepth == 0 && this.arrayDepth == 0) {
result.add(this.tokenBuffer);
this.tokenBuffer = new TokenBuffer(this.parser);
}
@@ -156,8 +185,4 @@ class Jackson2Tokenizer implements Function<DataBuffer, Flux<TokenBuffer>> {
(token == JsonToken.END_ARRAY && this.arrayDepth == 0));
}
public void endOfInput() {
this.inputFeeder.endOfInput();
}
}

79
spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java
View File

@@ -22,9 +22,9 @@ import java.util.List;
import java.util.function.Consumer;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.TreeNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.util.TokenBuffer;
import org.json.JSONException;
import org.junit.Before;
import org.junit.Test;
@@ -43,44 +43,39 @@ import static java.util.Collections.singletonList;
*/
public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase {
private JsonParser jsonParser;
private Jackson2Tokenizer tokenizer;
private ObjectMapper objectMapper;
private JsonFactory jsonFactory;
@Before
public void createParser() throws IOException {
JsonFactory factory = new JsonFactory();
this.jsonParser = factory.createNonBlockingByteArrayParser();
this.objectMapper = new ObjectMapper(factory);
jsonFactory = new JsonFactory();
this.objectMapper = new ObjectMapper(jsonFactory);
}
@Test
public void doNotTokenizeArrayElements() {
this.tokenizer = new Jackson2Tokenizer(this.jsonParser, false);
testTokenize(
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"),
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"));
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"), false);
testTokenize(
asList("{\"foo\": \"foofoo\"",
", \"bar\": \"barbar\"}"),
singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"));
singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"), false);
testTokenize(
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"),
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"));
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), false);
testTokenize(
singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"),
singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"));
singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"), false);
testTokenize(
asList("[{\"foo\": \"foofoo\", \"bar\"",
": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"),
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"));
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), false);
testTokenize(
asList("[",
@@ -90,31 +85,43 @@ public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase
",",
"{\"id\":3,\"name\":\"Ford\"}",
"]"),
singletonList("[{\"id\":1,\"name\":\"Robert\"},{\"id\":2,\"name\":\"Raide\"},{\"id\":3,\"name\":\"Ford\"}]"));
singletonList("[{\"id\":1,\"name\":\"Robert\"},{\"id\":2,\"name\":\"Raide\"},{\"id\":3,\"name\":\"Ford\"}]"), false);
// SPR-16166: top-level JSON values
testTokenize(asList("\"foo", "bar\"")
,singletonList("\"foobar\""), false);
testTokenize(asList("12", "34")
,singletonList("1234"), false);
testTokenize(asList("12.", "34")
,singletonList("12.34"), false);
// note that we do not test for null, true, or false, which are also valid top-level values,
// but are unsupported by JSONassert
}
@Test
public void tokenizeArrayElements() {
this.tokenizer = new Jackson2Tokenizer(this.jsonParser, true);
testTokenize(
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"),
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"));
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"), true);
testTokenize(
asList("{\"foo\": \"foofoo\"",
", \"bar\": \"barbar\"}"),
singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"));
singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"), true);
testTokenize(
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"),
asList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}",
"{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"));
"{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"), true);
testTokenize(
singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"),
asList("{\"foo\": \"bar\"}",
"{\"foo\": \"baz\"}"));
"{\"foo\": \"baz\"}"), true);
// SPR-15803: nested array
testTokenize(
@@ -126,19 +133,19 @@ public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase
asList(
"{\"id\":\"0\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}",
"{\"id\":\"1\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}",
"{\"id\":\"2\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}")
);
"{\"id\":\"2\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}"),
true);
// SPR-15803: nested array, no top-level array
testTokenize(
singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}"),
singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}"));
singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}"), true);
testTokenize(
asList("[{\"foo\": \"foofoo\", \"bar\"",
": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"),
asList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}",
"{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"));
"{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"), true);
testTokenize(
asList("[",
@@ -150,15 +157,27 @@ public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase
"]"),
asList("{\"id\":1,\"name\":\"Robert\"}",
"{\"id\":2,\"name\":\"Raide\"}",
"{\"id\":3,\"name\":\"Ford\"}"));
"{\"id\":3,\"name\":\"Ford\"}"), true);
// SPR-16166: top-level JSON values
testTokenize(asList("\"foo", "bar\"")
,singletonList("\"foobar\""), true);
testTokenize(asList("12", "34")
,singletonList("1234"), true);
testTokenize(asList("12.", "34")
,singletonList("12.34"), true);
}
private void testTokenize(List<String> source, List<String> expected) {
private void testTokenize(List<String> source, List<String> expected, boolean tokenizeArrayElements) {
Flux<DataBuffer> sourceFlux = Flux.fromIterable(source)
.map(this::stringBuffer);
Flux<String> result = sourceFlux
.flatMap(this.tokenizer)
Flux<TokenBuffer> tokenBufferFlux =
Jackson2Tokenizer.tokenize(sourceFlux, jsonFactory, tokenizeArrayElements);
Flux<String> result = tokenBufferFlux
.map(tokenBuffer -> {
try {
TreeNode root = this.objectMapper.readTree(tokenBuffer.asParser());