diff --git a/spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java b/spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java index a507098b44..3c233eaf2d 100644 --- a/spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java +++ b/spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java @@ -17,12 +17,10 @@ package org.springframework.http.codec.json; import java.io.IOException; -import java.io.UncheckedIOException; import java.lang.annotation.Annotation; import java.util.Map; import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; @@ -89,15 +87,9 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple } private Flux tokenize(Publisher input, boolean tokenizeArrayElements) { - try { - JsonFactory factory = getObjectMapper().getFactory(); - JsonParser parser = factory.createNonBlockingByteArrayParser(); - Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, tokenizeArrayElements); - return Flux.from(input).flatMap(tokenizer).doFinally(t -> tokenizer.endOfInput()); - } - catch (IOException ex) { - return Flux.error(new UncheckedIOException(ex)); - } + Flux inputFlux = Flux.from(input); + JsonFactory factory = getObjectMapper().getFactory(); + return Jackson2Tokenizer.tokenize(inputFlux, factory, tokenizeArrayElements); } private Flux decodeInternal(Flux tokens, ResolvableType elementType, diff --git a/spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java b/spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java index 9f6248a2ab..0cd3b78aed 100644 --- a/spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java +++ b/spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java @@ -21,11 +21,13 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Function; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.core.async.ByteArrayFeeder; import com.fasterxml.jackson.databind.util.TokenBuffer; +import org.jetbrains.annotations.NotNull; import reactor.core.publisher.Flux; import org.springframework.core.codec.DecodingException; @@ -41,7 +43,7 @@ import org.springframework.util.Assert; * @author Arjen Poutsma * @since 5.0 */ -class Jackson2Tokenizer implements Function> { +class Jackson2Tokenizer { private final JsonParser parser; @@ -57,15 +59,7 @@ class Jackson2Tokenizer implements Function> { private final ByteArrayFeeder inputFeeder; - /** - * Create a new instance of the {@code Jackson2Tokenizer}. - * @param parser the non-blocking parser, obtained via - * {@link com.fasterxml.jackson.core.JsonFactory#createNonBlockingByteArrayParser} - * @param tokenizeArrayElements if {@code true} and the "top level" JSON - * object is an array, each element is returned individually, immediately - * after it is received. - */ - public Jackson2Tokenizer(JsonParser parser, boolean tokenizeArrayElements) { + private Jackson2Tokenizer(JsonParser parser, boolean tokenizeArrayElements) { Assert.notNull(parser, "'parser' must not be null"); this.parser = parser; @@ -74,42 +68,78 @@ class Jackson2Tokenizer implements Function> { this.inputFeeder = (ByteArrayFeeder) this.parser.getNonBlockingInputFeeder(); } + /** + * Tokenize the given {@link DataBuffer} flux into a {@link TokenBuffer} flux, given the + * parameters. + * @param dataBuffers the source data buffers + * @param jsonFactory the factory to use + * @param tokenizeArrayElements if {@code true} and the "top level" JSON + * object is an array, each element is returned individually, immediately + * after it is received. + * @return the result token buffers + */ + public static Flux tokenize(Flux dataBuffers, JsonFactory jsonFactory, + boolean tokenizeArrayElements) { + try { + Jackson2Tokenizer tokenizer = + new Jackson2Tokenizer(jsonFactory.createNonBlockingByteArrayParser(), + tokenizeArrayElements); + return dataBuffers.flatMap(tokenizer::tokenize, Flux::error, tokenizer::endOfInput); + } + catch (IOException ex) { + return Flux.error(ex); + } + } - @Override - public Flux apply(DataBuffer dataBuffer) { + private Flux tokenize(DataBuffer dataBuffer) { byte[] bytes = new byte[dataBuffer.readableByteCount()]; dataBuffer.read(bytes); DataBufferUtils.release(dataBuffer); try { this.inputFeeder.feedInput(bytes, 0, bytes.length); - List result = new ArrayList<>(); - - while (true) { - JsonToken token = this.parser.nextToken(); - if (token == JsonToken.NOT_AVAILABLE) { - break; - } - updateDepth(token); - - if (!this.tokenizeArrayElements) { - processTokenNormal(token, result); - } - else { - processTokenArray(token, result); - } - } - return Flux.fromIterable(result); + return parseTokenBufferFlux(); } catch (JsonProcessingException ex) { return Flux.error(new DecodingException( "JSON decoding error: " + ex.getOriginalMessage(), ex)); } - catch (Exception ex) { + catch (IOException ex) { return Flux.error(ex); } } + private Flux endOfInput() { + this.inputFeeder.endOfInput(); + try { + return parseTokenBufferFlux(); + } + catch (IOException ex) { + return Flux.error(ex); + } + } + + @NotNull + private Flux parseTokenBufferFlux() throws IOException { + List result = new ArrayList<>(); + + while (true) { + JsonToken token = this.parser.nextToken(); + if (token == null || token == JsonToken.NOT_AVAILABLE) { + break; + } + updateDepth(token); + + if (!this.tokenizeArrayElements) { + processTokenNormal(token, result); + } + else { + processTokenArray(token, result); + } + } + return Flux.fromIterable(result); + } + private void updateDepth(JsonToken token) { switch (token) { case START_OBJECT: @@ -130,11 +160,10 @@ class Jackson2Tokenizer implements Function> { private void processTokenNormal(JsonToken token, List result) throws IOException { this.tokenBuffer.copyCurrentEvent(this.parser); - if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) { - if (this.objectDepth == 0 && this.arrayDepth == 0) { - result.add(this.tokenBuffer); - this.tokenBuffer = new TokenBuffer(this.parser); - } + if ((token.isStructEnd() || token.isScalarValue()) && + this.objectDepth == 0 && this.arrayDepth == 0) { + result.add(this.tokenBuffer); + this.tokenBuffer = new TokenBuffer(this.parser); } } @@ -144,8 +173,8 @@ class Jackson2Tokenizer implements Function> { this.tokenBuffer.copyCurrentEvent(this.parser); } - if (token == JsonToken.END_OBJECT && this.objectDepth == 0 && - (this.arrayDepth == 1 || this.arrayDepth == 0)) { + if ((token == JsonToken.END_OBJECT && this.objectDepth == 0 && (this.arrayDepth == 1 || this.arrayDepth == 0)) || + (token.isScalarValue()) && this.objectDepth == 0 && this.arrayDepth == 0) { result.add(this.tokenBuffer); this.tokenBuffer = new TokenBuffer(this.parser); } @@ -156,8 +185,4 @@ class Jackson2Tokenizer implements Function> { (token == JsonToken.END_ARRAY && this.arrayDepth == 0)); } - public void endOfInput() { - this.inputFeeder.endOfInput(); - } - } diff --git a/spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java b/spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java index 0f5633ef4f..572a1de90e 100644 --- a/spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java +++ b/spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java @@ -22,9 +22,9 @@ import java.util.List; import java.util.function.Consumer; import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.util.TokenBuffer; import org.json.JSONException; import org.junit.Before; import org.junit.Test; @@ -43,44 +43,39 @@ import static java.util.Collections.singletonList; */ public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase { - private JsonParser jsonParser; - - private Jackson2Tokenizer tokenizer; - private ObjectMapper objectMapper; + private JsonFactory jsonFactory; + @Before public void createParser() throws IOException { - JsonFactory factory = new JsonFactory(); - this.jsonParser = factory.createNonBlockingByteArrayParser(); - this.objectMapper = new ObjectMapper(factory); + jsonFactory = new JsonFactory(); + this.objectMapper = new ObjectMapper(jsonFactory); } @Test public void doNotTokenizeArrayElements() { - this.tokenizer = new Jackson2Tokenizer(this.jsonParser, false); - testTokenize( singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"), - singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}")); + singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"), false); testTokenize( asList("{\"foo\": \"foofoo\"", ", \"bar\": \"barbar\"}"), - singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}")); + singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"), false); testTokenize( singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), - singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]")); + singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), false); testTokenize( singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"), - singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]")); + singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"), false); testTokenize( asList("[{\"foo\": \"foofoo\", \"bar\"", ": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), - singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]")); + singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), false); testTokenize( asList("[", @@ -90,31 +85,43 @@ public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase ",", "{\"id\":3,\"name\":\"Ford\"}", "]"), - singletonList("[{\"id\":1,\"name\":\"Robert\"},{\"id\":2,\"name\":\"Raide\"},{\"id\":3,\"name\":\"Ford\"}]")); + singletonList("[{\"id\":1,\"name\":\"Robert\"},{\"id\":2,\"name\":\"Raide\"},{\"id\":3,\"name\":\"Ford\"}]"), false); + + // SPR-16166: top-level JSON values + testTokenize(asList("\"foo", "bar\"") + ,singletonList("\"foobar\""), false); + + testTokenize(asList("12", "34") + ,singletonList("1234"), false); + + testTokenize(asList("12.", "34") + ,singletonList("12.34"), false); + + // note that we do not test for null, true, or false, which are also valid top-level values, + // but are unsupported by JSONassert + } @Test public void tokenizeArrayElements() { - this.tokenizer = new Jackson2Tokenizer(this.jsonParser, true); - testTokenize( singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"), - singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}")); + singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"), true); testTokenize( asList("{\"foo\": \"foofoo\"", ", \"bar\": \"barbar\"}"), - singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}")); + singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"), true); testTokenize( singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), asList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}", - "{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}")); + "{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"), true); testTokenize( singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"), asList("{\"foo\": \"bar\"}", - "{\"foo\": \"baz\"}")); + "{\"foo\": \"baz\"}"), true); // SPR-15803: nested array testTokenize( @@ -126,19 +133,19 @@ public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase asList( "{\"id\":\"0\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}", "{\"id\":\"1\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}", - "{\"id\":\"2\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}") - ); + "{\"id\":\"2\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}"), + true); // SPR-15803: nested array, no top-level array testTokenize( singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}"), - singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}")); + singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}"), true); testTokenize( asList("[{\"foo\": \"foofoo\", \"bar\"", ": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), asList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}", - "{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}")); + "{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"), true); testTokenize( asList("[", @@ -150,15 +157,27 @@ public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase "]"), asList("{\"id\":1,\"name\":\"Robert\"}", "{\"id\":2,\"name\":\"Raide\"}", - "{\"id\":3,\"name\":\"Ford\"}")); + "{\"id\":3,\"name\":\"Ford\"}"), true); + + // SPR-16166: top-level JSON values + testTokenize(asList("\"foo", "bar\"") + ,singletonList("\"foobar\""), true); + + testTokenize(asList("12", "34") + ,singletonList("1234"), true); + + testTokenize(asList("12.", "34") + ,singletonList("12.34"), true); } - private void testTokenize(List source, List expected) { + private void testTokenize(List source, List expected, boolean tokenizeArrayElements) { Flux sourceFlux = Flux.fromIterable(source) .map(this::stringBuffer); - Flux result = sourceFlux - .flatMap(this.tokenizer) + Flux tokenBufferFlux = + Jackson2Tokenizer.tokenize(sourceFlux, jsonFactory, tokenizeArrayElements); + + Flux result = tokenBufferFlux .map(tokenBuffer -> { try { TreeNode root = this.objectMapper.readTree(tokenBuffer.asParser());