Process tokens after each feed in Jackson2Tokenizer

This commit ensures that we process after each fed buffer in
Jackson2Tokenizer, instead of after all fed buffers.

Closes gh-31747

spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java

@@ -91,10 +91,12 @@ final class Jackson2Tokenizer {
 	private List<TokenBuffer> tokenize(DataBuffer dataBuffer) {
 		try {
 			int bufferSize = dataBuffer.readableByteCount();
+			List<TokenBuffer> tokens = new ArrayList<>();
 			if (this.inputFeeder instanceof ByteBufferFeeder byteBufferFeeder) {
 				try (DataBuffer.ByteBufferIterator iterator = dataBuffer.readableByteBuffers()) {
 					while (iterator.hasNext()) {
 						byteBufferFeeder.feedInput(iterator.next());
+						parseTokens(tokens);
 					}
 				}
 			}
@@ -102,10 +104,10 @@ final class Jackson2Tokenizer {
 				byte[] bytes = new byte[bufferSize];
 				dataBuffer.read(bytes);
 				byteArrayFeeder.feedInput(bytes, 0, bufferSize);
+				parseTokens(tokens);
 			}
-			List<TokenBuffer> result = parseTokenBufferFlux();
-			assertInMemorySize(bufferSize, result);
-			return result;
+			assertInMemorySize(bufferSize, tokens);
+			return tokens;
 		}
 		catch (JsonProcessingException ex) {
 			throw new DecodingException("JSON decoding error: " + ex.getOriginalMessage(), ex);
@@ -122,7 +124,9 @@ final class Jackson2Tokenizer {
 		return Flux.defer(() -> {
 			this.inputFeeder.endOfInput();
 			try {
-				return Flux.fromIterable(parseTokenBufferFlux());
+				List<TokenBuffer> tokens = new ArrayList<>();
+				parseTokens(tokens);
+				return Flux.fromIterable(tokens);
 			}
 			catch (JsonProcessingException ex) {
 				throw new DecodingException("JSON decoding error: " + ex.getOriginalMessage(), ex);
@@ -133,9 +137,7 @@ final class Jackson2Tokenizer {
 		});
 	}
 
-	private List<TokenBuffer> parseTokenBufferFlux() throws IOException {
-		List<TokenBuffer> result = new ArrayList<>();
-
+	private void parseTokens(List<TokenBuffer> tokens) throws IOException {
 		// SPR-16151: Smile data format uses null to separate documents
 		boolean previousNull = false;
 		while (!this.parser.isClosed()) {
@@ -153,13 +155,12 @@ final class Jackson2Tokenizer {
 			}
 			updateDepth(token);
 			if (!this.tokenizeArrayElements) {
-				processTokenNormal(token, result);
+				processTokenNormal(token, tokens);
 			}
 			else {
-				processTokenArray(token, result);
+				processTokenArray(token, tokens);
 			}
 		}
-		return result;
 	}
 
 	private void updateDepth(JsonToken token) {