From dc6a2c93b842b11575712b61838a5dff7f518049 Mon Sep 17 00:00:00 2001 From: Jonathan Gray Date: Wed, 24 Jun 2015 01:57:48 +0100 Subject: [PATCH] BATCH-2402 Refactored (behaviour is exactly the same) to avoid temporary String allocation in order to be more memory space efficient. This helps to avoid garbage collection with large numbers of batch lines and large numbers of quoted tokens. --- .../transform/DelimitedLineTokenizer.java | 137 ++++++++++-------- 1 file changed, 78 insertions(+), 59 deletions(-) diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java index bcc44c7b3..5ebb519b6 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java @@ -34,7 +34,7 @@ import org.springframework.util.StringUtils; * @author Dave Syer * @author Michael Minella */ -public class DelimitedLineTokenizer extends AbstractLineTokenizer +public class DelimitedLineTokenizer extends AbstractLineTokenizer implements InitializingBean { /** * Convenient constant for the common case of a tab delimiter. @@ -59,6 +59,8 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer private String quoteString; + private String escapedQuoteString; + private Collection includedFields = null; /** @@ -123,6 +125,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer public void setQuoteCharacter(char quoteCharacter) { this.quoteCharacter = quoteCharacter; this.quoteString = "" + quoteCharacter; + this.escapedQuoteString = "" + quoteCharacter + quoteCharacter; } /** @@ -151,7 +154,7 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer char currentChar = chars[i]; boolean isEnd = (i == (length - 1)); - boolean isDelimiter = isDelimiter(chars, i, delimiter, endIndexLastDelimiter); + boolean isDelimiter = endsWithDelimiter(chars, i, endIndexLastDelimiter); if ((isDelimiter && !inQuoted) || isEnd) { endIndexLastDelimiter = i; @@ -165,7 +168,8 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer } if (includedFields == null || includedFields.contains(fieldCount)) { - String value = maybeStripQuotes(new String(chars, lastCut, endPosition)); + String value = + substringWithTrimmedWhitespaceAndQuotesIfQuotesPresent(chars, lastCut, endPosition); tokens.add(value); } @@ -189,65 +193,80 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer return tokens; } - /** - * If the string is quoted strip (possibly with whitespace outside the - * quotes (which will be stripped), replace escaped quotes inside the - * string. Quotes are escaped with double instances of the quote character. - * - * @param string - * @return the same string but stripped and unescaped if necessary - */ - private String maybeStripQuotes(String string) { - String value = string.trim(); - if (isQuoted(value)) { - value = StringUtils.replace(value, "" + quoteCharacter + quoteCharacter, "" + quoteCharacter); - int endLength = value.length() - 1; - // used to deal with empty quoted values - if (endLength == 0) { - endLength = 1; - } - value = value.substring(1, endLength); - return value; - } - return string; - } + /** + * Trim and leading or trailing quotes (and any leading or trailing + * whitespace before or after the quotes) from within the specified character + * array beginning at the specified offset index for the specified count. + *

+ * Quotes are escaped with double instances of the quote character. + * + * @param chars the character array + * @param offset index from which to begin extracting substring + * @param count length of substring + * @return a substring from the specified offset within the character array + * with any leading or trailing whitespace trimmed. + * @see String#trim() + */ + private String substringWithTrimmedWhitespaceAndQuotesIfQuotesPresent(char chars[], int offset, int count) { + int start = offset; + int len = count; - /** - * Is this string surrounded by quote characters? - * - * @param value - * @return true if the value starts and ends with the - * {@link #quoteCharacter} - */ - private boolean isQuoted(String value) { - if (value.startsWith(quoteString) && value.endsWith(quoteString)) { - return true; - } - return false; - } + while ((start < (start + len)) && (chars[start] <= ' ')) { + start++; + len--; + } - /** - * Is the supplied character the delimiter character? - * - * @param chars the character array to be checked - * @return true if the supplied character is the delimiter - * character - * @see DelimitedLineTokenizer#DelimitedLineTokenizer(String) - */ - private boolean isDelimiter(char[] chars, int i, String token, int endIndexLastDelimiter) { - boolean result = false; + while ((start < (start + len)) && ((start + len - 1 < chars.length) && (chars[start + len - 1] <= ' '))) { + len--; + } - if(i-endIndexLastDelimiter >= delimiter.length()) { - if(i >= token.length() - 1) { - String end = new String(chars, (i-token.length()) + 1, token.length()); - if(token.equals(end)) { - result = true; - } - } - } + String value; - return result; - } + if ((chars.length > 2) && (chars[start] == quoteCharacter) && (chars[start + len - 1] == quoteCharacter)) { + value = new String(chars, start + 1, len - 2); + if (value.contains(escapedQuoteString)) { + value = StringUtils.replace(value, escapedQuoteString, quoteString); + } + } + else { + value = new String(chars, offset, count); + } + + return value; + } + + /** + * Do the character(s) in the specified array end, at the specified end + * index, with the delimiter character(s)? + *

+ * Checks that the specified end index is sufficiently greater than the + * specified previous delimiter end index to warrant trying to match + * another delimiter. Also checks that the specified end index is + * sufficiently large to be able to match the length of a delimiter. + * + * @param chars the character array + * @param end the index in up to which the delimiter should be matched + * @param previous the index of the end of the last delimiter + * @return true if the character(s) from the specified end + * match the delimiter character(s), otherwise false + * @see DelimitedLineTokenizer#DelimitedLineTokenizer(String) + */ + private boolean endsWithDelimiter(char[] chars, int end, int previous) { + boolean result = false; + + if (end - previous >= delimiter.length()) { + if (end >= delimiter.length() - 1) { + result = true; + for (int j = 0; j < delimiter.length() && (((end - delimiter.length() + 1) + j) < chars.length); j++) { + if (delimiter.charAt(j) != chars[(end - delimiter.length() + 1) + j]) { + result = false; + } + } + } + } + + return result; + } /** * Is the supplied character a quote character? @@ -262,6 +281,6 @@ public class DelimitedLineTokenizer extends AbstractLineTokenizer @Override public void afterPropertiesSet() throws Exception { - Assert.state(null != delimiter && 0 != delimiter.length()); + Assert.state(null != delimiter && 0 != delimiter.length()); } }