Spring/spring-framework
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert non-UTF-8 JSON

Browse Source 
Jackson's asynchronous parser does not support any encoding except UTF-8
(or ASCII). This commit converts non-UTF-8/ASCII encoded JSON to UTF-8.

Closes gh-24489
This commit is contained in:
Arjen Poutsma
2020-02-20 10:48:41 +01:00
parent 4e55262521
commit 439ffe2e8a
3 changed files with 102 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java
View File

@@ -120,11 +120,29 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
forceUseOfBigDecimal = true;
}
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(Flux.from(input), this.jsonFactory, getObjectMapper(),
Flux<DataBuffer> processed = processInput(input, elementType, mimeType, hints);
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(processed, this.jsonFactory, getObjectMapper(),
true, forceUseOfBigDecimal, getMaxInMemorySize());
return decodeInternal(tokens, elementType, mimeType, hints);
}
/**
* Process the input publisher into a flux. Default implementation returns
* {@link Flux#from(Publisher)}, but subclasses can choose to to customize
* this behaviour.
* @param input the {@code DataBuffer} input stream to process
* @param elementType the expected type of elements in the output stream
* @param mimeType the MIME type associated with the input stream (optional)
* @param hints additional information about how to do encode
* @return the processed flux
* @since 5.1.14
*/
protected Flux<DataBuffer> processInput(Publisher<DataBuffer> input, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
return Flux.from(input);
}
@Override
public Mono<Object> decodeToMono(Publisher<DataBuffer> input, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
@@ -134,7 +152,8 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
forceUseOfBigDecimal = true;
}
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(Flux.from(input), this.jsonFactory, getObjectMapper(),
Flux<DataBuffer> processed = processInput(input, elementType, mimeType, hints);
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(processed, this.jsonFactory, getObjectMapper(),
false, forceUseOfBigDecimal, getMaxInMemorySize());
return decodeInternal(tokens, elementType, mimeType, hints).singleOrEmpty();
}

47
spring-web/src/main/java/org/springframework/http/codec/json/Jackson2JsonDecoder.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,10 +16,24 @@
package org.springframework.http.codec.json;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Map;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.reactivestreams.Publisher;
import reactor.core.publisher.Flux;
import org.springframework.core.ResolvableType;
import org.springframework.core.codec.StringDecoder;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferFactory;
import org.springframework.core.io.buffer.DefaultDataBufferFactory;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.lang.Nullable;
import org.springframework.util.MimeType;
import org.springframework.util.MimeTypeUtils;
/**
* Decode a byte stream into JSON and convert to Object's with Jackson 2.9,
@@ -32,6 +46,11 @@ import org.springframework.util.MimeType;
*/
public class Jackson2JsonDecoder extends AbstractJackson2Decoder {
private static final StringDecoder STRING_DECODER = StringDecoder.textPlainOnly(Arrays.asList(",", "\n"), false);
private static final ResolvableType STRING_TYPE = ResolvableType.forClass(String.class);
public Jackson2JsonDecoder() {
super(Jackson2ObjectMapperBuilder.json().build());
}
@@ -40,4 +59,28 @@ public class Jackson2JsonDecoder extends AbstractJackson2Decoder {
super(mapper, mimeTypes);
}
@Override
protected Flux<DataBuffer> processInput(Publisher<DataBuffer> input, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
Flux<DataBuffer> flux = Flux.from(input);
if (mimeType == null) {
return flux;
}
// Jackson asynchronous parser only supports UTF-8
Charset charset = mimeType.getCharset();
if (charset == null || StandardCharsets.UTF_8.equals(charset) || StandardCharsets.US_ASCII.equals(charset)) {
return flux;
}
// Potentially, the memory consumption of this conversion could be improved by using CharBuffers instead
// of allocating Strings, but that would require refactoring the buffer tokenization code from StringDecoder
MimeType textMimeType = new MimeType(MimeTypeUtils.TEXT_PLAIN, charset);
Flux<String> decoded = STRING_DECODER.decode(input, STRING_TYPE, textMimeType, null);
DataBufferFactory factory = new DefaultDataBufferFactory();
return decoded.map(s -> factory.wrap(s.getBytes(StandardCharsets.UTF_8)));
}
}