Optimize QueryTokenStream handling.

This commit moves EQL and HQL parsers to QueryTokenStream and optimizes composite renderers as well as JSQLParser usage.
We also avoid Single-thread construction on each parse run to avoid thread creation overhead.

See: #3309

spring-data-jpa-performance/src/test/java/org/springframework/data/jpa/repository/Profiler.java

@@ -1,52 +0,0 @@
-/*
- * Copyright 2024 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.springframework.data.jpa.repository;
-
-import java.lang.reflect.Method;
-
-import org.springframework.data.domain.Sort;
-import org.springframework.data.jpa.provider.PersistenceProvider;
-import org.springframework.data.jpa.repository.query.JpaQueryMethod;
-import org.springframework.data.jpa.repository.query.StringQuery;
-import org.springframework.data.projection.ProjectionFactory;
-import org.springframework.data.projection.SpelAwareProxyProjectionFactory;
-import org.springframework.data.repository.core.support.DefaultRepositoryMetadata;
-
-/**
- * @author Mark Paluch
- */
-public class Profiler {
-
-	public static void main(String[] args) throws Exception {
-
-		DefaultRepositoryMetadata art = new DefaultRepositoryMetadata(PersonRepository.class);
-		Method method = PersonRepository.class.getMethod("findAllWithAnnotatedQueryByFirstname", String.class, Sort.class);
-		ProjectionFactory projectionFactory = new SpelAwareProxyProjectionFactory();
-
-		System.out.println("Ready. Waiting 10sec");
-		Thread.sleep(10000);
-
-		System.out.println("Go!");
-
-
-		while (true) {
-
-			JpaQueryMethod queryMethod = new JpaQueryMethod(method, art, projectionFactory, PersistenceProvider.HIBERNATE);
-			StringQuery stringQuery = new StringQuery(queryMethod.getRequiredAnnotatedQuery(), false);
-		}
-
-	}
-}

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/EmptyQueryTokenStream.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.jpa.repository.query;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+/**
+ * Empty QueryTokenStream.
+ *
+ * @author Mark Paluch
+ * @since 3.4
+ */
+class EmptyQueryTokenStream implements QueryTokenStream {
+
+	static final EmptyQueryTokenStream INSTANCE = new EmptyQueryTokenStream();
+
+	private EmptyQueryTokenStream() {}
+
+	@Override
+	public QueryToken getFirst() {
+		return null;
+	}
+
+	@Override
+	public QueryToken getLast() {
+		return null;
+	}
+
+	@Override
+	public boolean isExpression() {
+		return false;
+	}
+
+	@Override
+	public int size() {
+		return 0;
+	}
+
+	@Override
+	public boolean isEmpty() {
+		return true;
+	}
+
+	@Override
+	public Iterator<QueryToken> iterator() {
+		return Collections.emptyIterator();
+	}
+}

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/EqlCountQueryTransformer.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import org.springframework.data.jpa.repository.query.QueryRenderer.QueryRendererBuilder;
 import org.springframework.data.jpa.repository.query.QueryTransformers.CountSelectionTokenStream;
@@ -62,28 +62,28 @@ class EqlCountQueryTransformer extends EqlQueryRenderer {
 	}
 
 	@Override
-	public QueryRendererBuilder visitSelect_clause(EqlParser.Select_clauseContext ctx) {
+	public QueryTokenStream visitSelect_clause(EqlParser.Select_clauseContext ctx) {
 
 		QueryRendererBuilder builder = QueryRenderer.builder();
 
-		builder.append(JpaQueryParsingToken.expression(ctx.SELECT()));
+		builder.append(QueryTokens.expression(ctx.SELECT()));
 		builder.append(TOKEN_COUNT_FUNC);
 
 		if (countProjection != null) {
-			builder.append(JpaQueryParsingToken.token(countProjection));
+			builder.append(QueryTokens.token(countProjection));
 		}
 
 		QueryRendererBuilder nested = QueryRenderer.builder();
 
 		if (ctx.DISTINCT() != null) {
-			nested.append(JpaQueryParsingToken.expression(ctx.DISTINCT()));
+			nested.append(QueryTokens.expression(ctx.DISTINCT()));
 		}
 
 		if (countProjection == null) {
 
 			if (ctx.DISTINCT() != null) {
 
-				QueryRendererBuilder selectionListbuilder = QueryRendererBuilder.concat(ctx.select_item(), this::visit,
+				QueryTokenStream selectionListbuilder = QueryTokenStream.concat(ctx.select_item(), this::visit,
 						TOKEN_COMMA);
 
 				CountSelectionTokenStream countSelection = QueryTransformers
@@ -91,13 +91,13 @@ class EqlCountQueryTransformer extends EqlQueryRenderer {
 
 				if (countSelection.requiresPrimaryAlias()) {
 					// constructor
-					nested.append(new JpaQueryParsingToken(primaryFromAlias));
+					nested.append(QueryTokens.token(primaryFromAlias));
 				} else {
 					// keep all the select items to distinct against
 					nested.append(countSelection);
 				}
 			} else {
-				nested.append(new JpaQueryParsingToken(primaryFromAlias));
+				nested.append(QueryTokens.token(primaryFromAlias));
 			}
 		}
 

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/EqlQueryIntrospector.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -34,7 +34,7 @@ class EqlQueryIntrospector extends EqlBaseVisitor<Void> implements ParsedQueryIn
 	private final EqlQueryRenderer renderer = new EqlQueryRenderer();
 
 	private @Nullable String primaryFromAlias = null;
-	private @Nullable List<JpaQueryParsingToken> projection;
+	private @Nullable List<QueryToken> projection;
 	private boolean projectionProcessed;
 	private boolean hasConstructorExpression = false;
 
@@ -44,7 +44,7 @@ class EqlQueryIntrospector extends EqlBaseVisitor<Void> implements ParsedQueryIn
 	}
 
 	@Override
-	public List<JpaQueryParsingToken> getProjection() {
+	public List<QueryToken> getProjection() {
 		return projection == null ? Collections.emptyList() : projection;
 	}
 
@@ -57,7 +57,7 @@ class EqlQueryIntrospector extends EqlBaseVisitor<Void> implements ParsedQueryIn
 	public Void visitSelect_clause(EqlParser.Select_clauseContext ctx) {
 
 		List<EqlParser.Select_itemContext> selections = ctx.select_item();
-		List<JpaQueryParsingToken> selectItemTokens = new ArrayList<>(selections.size() * 2);
+		List<QueryToken> selectItemTokens = new ArrayList<>(selections.size() * 2);
 
 		for (EqlParser.Select_itemContext selection : selections) {
 
@@ -65,7 +65,7 @@ class EqlQueryIntrospector extends EqlBaseVisitor<Void> implements ParsedQueryIn
 				selectItemTokens.add(TOKEN_COMMA);
 			}
 
-			selectItemTokens.add(JpaQueryParsingToken.token(renderer.visitSelect_item(selection).build().render()));
+			selectItemTokens.add(QueryTokens.token(QueryRenderer.from(renderer.visitSelect_item(selection)).render()));
 		}
 
 		if (!projectionProcessed) {

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/EqlSortedQueryTransformer.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import java.util.List;
 
@@ -81,7 +81,7 @@ class EqlSortedQueryTransformer extends EqlQueryRenderer {
 	private void doVisitOrderBy(QueryRendererBuilder builder, EqlParser.Select_statementContext ctx) {
 
 		if (ctx.orderby_clause() != null) {
-			QueryRendererBuilder existingOrder = visit(ctx.orderby_clause());
+			QueryTokenStream existingOrder = visit(ctx.orderby_clause());
 			if (sort.isSorted()) {
 				builder.appendInline(existingOrder);
 			} else {
@@ -91,7 +91,7 @@ class EqlSortedQueryTransformer extends EqlQueryRenderer {
 
 		if (sort.isSorted()) {
 
-			List<JpaQueryParsingToken> sortBy = transformerSupport.orderBy(primaryFromAlias, sort);
+			List<QueryToken> sortBy = transformerSupport.orderBy(primaryFromAlias, sort);
 
 			if (ctx.orderby_clause() != null) {
 
@@ -106,24 +106,27 @@ class EqlSortedQueryTransformer extends EqlQueryRenderer {
 	}
 
 	@Override
-	public QueryRendererBuilder visitSelect_item(EqlParser.Select_itemContext ctx) {
+	public QueryTokenStream visitSelect_item(EqlParser.Select_itemContext ctx) {
 
-		QueryRendererBuilder builder = super.visitSelect_item(ctx);
+		QueryTokenStream tokens = super.visitSelect_item(ctx);
 
-		if (ctx.result_variable() != null) {
-			transformerSupport.registerAlias(builder.lastToken());
+		if (ctx.result_variable() != null && !tokens.isEmpty()) {
+			transformerSupport.registerAlias(tokens.getLast());
 		}
 
-		return builder;
+		return tokens;
 	}
 
 	@Override
-	public QueryRendererBuilder visitJoin(EqlParser.JoinContext ctx) {
+	public QueryTokenStream visitJoin(EqlParser.JoinContext ctx) {
 
-		QueryRendererBuilder builder = super.visitJoin(ctx);
-		transformerSupport.registerAlias(builder.lastToken());
+		QueryTokenStream tokens = super.visitJoin(ctx);
 
-		return builder;
+		if (!tokens.isEmpty()) {
+			transformerSupport.registerAlias(tokens.getLast());
+		}
+
+		return tokens;
 	}
 
 }

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/HqlCountQueryTransformer.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import org.springframework.data.jpa.repository.query.QueryRenderer.QueryRendererBuilder;
 import org.springframework.data.jpa.repository.query.QueryTransformers.CountSelectionTokenStream;
@@ -76,12 +76,12 @@ class HqlCountQueryTransformer extends HqlQueryRenderer {
 			countBuilder.append(TOKEN_SELECT_COUNT);
 
 			if (countProjection != null) {
-				countBuilder.append(new JpaQueryParsingToken(countProjection));
+				countBuilder.append(QueryTokens.token(countProjection));
 			} else {
 				if (primaryFromAlias == null) {
 					countBuilder.append(TOKEN_DOUBLE_UNDERSCORE);
 				} else {
-					countBuilder.append(JpaQueryParsingToken.token(primaryFromAlias));
+					countBuilder.append(QueryTokens.token(primaryFromAlias));
 				}
 			}
 
@@ -133,7 +133,7 @@ class HqlCountQueryTransformer extends HqlQueryRenderer {
 		} else if (ctx.subquery() != null) {
 
 			if (ctx.LATERAL() != null) {
-				builder.append(JpaQueryParsingToken.expression(ctx.LATERAL()));
+				builder.append(QueryTokens.expression(ctx.LATERAL()));
 			}
 
 			QueryRendererBuilder nested = QueryRenderer.builder();
@@ -158,7 +158,7 @@ class HqlCountQueryTransformer extends HqlQueryRenderer {
 		QueryRendererBuilder builder = QueryRenderer.builder();
 
 		builder.appendExpression(visit(ctx.joinType()));
-		builder.append(JpaQueryParsingToken.expression(ctx.JOIN()));
+		builder.append(QueryTokens.expression(ctx.JOIN()));
 
 		builder.appendExpression(visit(ctx.joinTarget()));
 
@@ -170,25 +170,25 @@ class HqlCountQueryTransformer extends HqlQueryRenderer {
 	}
 
 	@Override
-	public QueryRendererBuilder visitSelectClause(HqlParser.SelectClauseContext ctx) {
+	public QueryTokenStream visitSelectClause(HqlParser.SelectClauseContext ctx) {
 
 		QueryRendererBuilder builder = QueryRenderer.builder();
-		builder.append(JpaQueryParsingToken.expression(ctx.SELECT()));
+		builder.append(QueryTokens.expression(ctx.SELECT()));
 
-		QueryRendererBuilder selectionListbuilder = visit(ctx.selectionList());
+		QueryTokenStream selectionListbuilder = visit(ctx.selectionList());
 
 		if (!isSubquery(ctx)) {
 
 			builder.append(TOKEN_COUNT_FUNC);
 
 			if (countProjection != null) {
-				builder.append(JpaQueryParsingToken.token(countProjection));
+				builder.append(QueryTokens.token(countProjection));
 			}
 
 			QueryRendererBuilder nested = QueryRenderer.builder();
 
 			if (ctx.DISTINCT() != null) {
-				nested.append(JpaQueryParsingToken.expression(ctx.DISTINCT()));
+				nested.append(QueryTokens.expression(ctx.DISTINCT()));
 			}
 
 			if (countProjection == null) {
@@ -200,13 +200,13 @@ class HqlCountQueryTransformer extends HqlQueryRenderer {
 
 					if (countSelection.requiresPrimaryAlias()) {
 						// constructor
-						nested.append(new JpaQueryParsingToken(primaryFromAlias));
+						nested.append(QueryTokens.token(primaryFromAlias));
 					} else {
 						// keep all the select items to distinct against
 						nested.append(countSelection);
 					}
 				} else {
-					nested.append(new JpaQueryParsingToken(primaryFromAlias));
+					nested.append(QueryTokens.token(primaryFromAlias));
 				}
 			}
 
@@ -216,7 +216,7 @@ class HqlCountQueryTransformer extends HqlQueryRenderer {
 		} else {
 
 			if (ctx.DISTINCT() != null) {
-				builder.append(JpaQueryParsingToken.expression(ctx.DISTINCT()));
+				builder.append(QueryTokens.expression(ctx.DISTINCT()));
 			}
 
 			builder.append(selectionListbuilder);

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/HqlQueryIntrospector.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -34,7 +34,7 @@ class HqlQueryIntrospector extends HqlBaseVisitor<Void> implements ParsedQueryIn
 	private final HqlQueryRenderer renderer = new HqlQueryRenderer();
 
 	private @Nullable String primaryFromAlias = null;
-	private @Nullable List<JpaQueryParsingToken> projection;
+	private @Nullable List<QueryToken> projection;
 	private boolean projectionProcessed;
 	private boolean hasConstructorExpression = false;
 
@@ -44,7 +44,7 @@ class HqlQueryIntrospector extends HqlBaseVisitor<Void> implements ParsedQueryIn
 	}
 
 	@Override
-	public List<JpaQueryParsingToken> getProjection() {
+	public List<QueryToken> getProjection() {
 		return projection == null ? Collections.emptyList() : projection;
 	}
 
@@ -57,7 +57,7 @@ class HqlQueryIntrospector extends HqlBaseVisitor<Void> implements ParsedQueryIn
 	public Void visitSelectClause(HqlParser.SelectClauseContext ctx) {
 
 		List<HqlParser.SelectionContext> selections = ctx.selectionList().selection();
-		List<JpaQueryParsingToken> selectItemTokens = new ArrayList<>(selections.size() * 2);
+		List<QueryToken> selectItemTokens = new ArrayList<>(selections.size() * 2);
 
 		for (HqlParser.SelectionContext selection : selections) {
 
@@ -65,7 +65,7 @@ class HqlQueryIntrospector extends HqlBaseVisitor<Void> implements ParsedQueryIn
 				selectItemTokens.add(TOKEN_COMMA);
 			}
 
-			selectItemTokens.add(JpaQueryParsingToken.token(renderer.visitSelection(selection).build().render()));
+			selectItemTokens.add(QueryTokens.token(QueryRenderer.from(renderer.visitSelection(selection)).render()));
 		}
 
 		if (!projectionProcessed) {

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/HqlSortedQueryTransformer.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import java.util.List;
 
@@ -63,7 +63,7 @@ class HqlSortedQueryTransformer extends HqlQueryRenderer {
 		if (!isSubquery(ctx)) {
 
 			if (ctx.queryOrder() != null) {
-				QueryRendererBuilder existingOrder = visit(ctx.queryOrder());
+				QueryTokenStream existingOrder = visit(ctx.queryOrder());
 				if (sort.isSorted()) {
 					builder.appendInline(existingOrder);
 				} else {
@@ -73,7 +73,7 @@ class HqlSortedQueryTransformer extends HqlQueryRenderer {
 
 			if (sort.isSorted()) {
 
-				List<JpaQueryParsingToken> sortBy = transformerSupport.orderBy(primaryFromAlias, sort);
+				List<QueryToken> sortBy = transformerSupport.orderBy(primaryFromAlias, sort);
 
 				if (ctx.queryOrder() != null) {
 
@@ -96,39 +96,39 @@ class HqlSortedQueryTransformer extends HqlQueryRenderer {
 	}
 
 	@Override
-	public QueryRendererBuilder visitJoinPath(HqlParser.JoinPathContext ctx) {
+	public QueryTokenStream visitJoinPath(HqlParser.JoinPathContext ctx) {
 
-		QueryRendererBuilder builder = super.visitJoinPath(ctx);
+		QueryTokenStream tokens = super.visitJoinPath(ctx);
 
 		if (ctx.variable() != null) {
-			transformerSupport.registerAlias(builder.lastToken());
+			transformerSupport.registerAlias(tokens.getLast());
 		}
 
-		return builder;
+		return tokens;
 	}
 
 	@Override
-	public QueryRendererBuilder visitJoinSubquery(HqlParser.JoinSubqueryContext ctx) {
+	public QueryTokenStream visitJoinSubquery(HqlParser.JoinSubqueryContext ctx) {
 
-		QueryRendererBuilder builder = super.visitJoinSubquery(ctx);
+		QueryTokenStream tokens = super.visitJoinSubquery(ctx);
 
-		if (ctx.variable() != null) {
-			transformerSupport.registerAlias(builder.lastToken());
+		if (ctx.variable() != null && !tokens.isEmpty()) {
+			transformerSupport.registerAlias(tokens.getLast());
 		}
 
-		return builder;
+		return tokens;
 	}
 
 	@Override
-	public QueryRendererBuilder visitVariable(HqlParser.VariableContext ctx) {
+	public QueryTokenStream visitVariable(HqlParser.VariableContext ctx) {
 
-		QueryRendererBuilder builder = super.visitVariable(ctx);
+		QueryTokenStream tokens = super.visitVariable(ctx);
 
-		if (ctx.identifier() != null) {
-			transformerSupport.registerAlias(builder.lastToken());
+		if (ctx.identifier() != null && !tokens.isEmpty()) {
+			transformerSupport.registerAlias(tokens.getLast());
 		}
 
-		return builder;
+		return tokens;
 	}
 
 }

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/JSqlParserQueryEnhancer.java

@@ -18,11 +18,13 @@ package org.springframework.data.jpa.repository.query;
 import static org.springframework.data.jpa.repository.query.JSqlParserUtils.*;
 import static org.springframework.data.jpa.repository.query.QueryUtils.*;
 
-import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.expression.Alias;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.Function;
+import net.sf.jsqlparser.parser.CCJSqlParser;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.parser.ParseException;
+import net.sf.jsqlparser.parser.feature.Feature;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.statement.Statement;
 import net.sf.jsqlparser.statement.delete.Delete;
@@ -99,11 +101,25 @@ public class JSqlParserQueryEnhancer implements QueryEnhancer {
 	 * @param query the query to parse
 	 * @return the parsed query
 	 */
-	private static <T extends Statement> T parseStatement(String query, Class<T> classOfT) {
+	static <T extends Statement> T parseStatement(String sql, Class<T> classOfT) {
 
 		try {
-			return classOfT.cast(CCJSqlParserUtil.parse(query));
-		} catch (JSQLParserException e) {
+
+			CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+			boolean allowComplex = parser.getConfiguration().getAsBoolean(Feature.allowComplexParsing);
+			try {
+				return classOfT.cast(parser.withAllowComplexParsing(true).Statement());
+			} catch (ParseException ex) {
+				if (allowComplex && CCJSqlParserUtil.getNestingDepth(sql) <= CCJSqlParserUtil.ALLOWED_NESTING_DEPTH) {
+					// beware: the parser must not be reused, but needs to be re-initiated
+					parser = CCJSqlParserUtil.newParser(sql);
+					return classOfT.cast(parser.withAllowComplexParsing(true).Statement());
+				} else {
+					throw ex;
+				}
+			}
+
+		} catch (ParseException e) {
 			throw new IllegalArgumentException("The query you provided is not a valid SQL Query", e);
 		}
 	}

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/JpaQueryEnhancer.java

@@ -61,8 +61,8 @@ class JpaQueryEnhancer implements QueryEnhancer {
 		this.countQueryFunction = countQueryFunction;
 		this.introspector.visit(context);
 
-		List<JpaQueryParsingToken> tokens = introspector.getProjection();
-		this.projection = tokens.isEmpty() ? "" : QueryRenderer.TokenRenderer.render(tokens);
+		List<QueryToken> tokens = introspector.getProjection();
+		this.projection = tokens.isEmpty() ? "" : new QueryRenderer.TokenRenderer(tokens).render();
 	}
 
 	static <P extends Parser> ParserRuleContext parse(String query, Function<CharStream, Lexer> lexerFactoryFunction,
@@ -176,7 +176,7 @@ class JpaQueryEnhancer implements QueryEnhancer {
 	}
 
 	/**
-	 * Look up the {@link DeclaredQuery} from the {@link JpaQueryParser}.
+	 * Look up the {@link DeclaredQuery} from the query parser.
 	 */
 	@Override
 	public DeclaredQuery getQuery() {

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/JpaQueryParsingToken.java

@@ -1,135 +0,0 @@
-/*
- * Copyright 2022-2024 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.springframework.data.jpa.repository.query;
-
-import java.util.function.Supplier;
-
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.tree.TerminalNode;
-
-/**
- * A value type used to represent a JPA query token. NOTE: Sometimes the token's value is based upon a value found later
- * in the parsing process, so the text itself is wrapped in a {@link Supplier}.
- *
- * @author Greg Turnquist
- * @author Christoph Strobl
- * @since 3.1
- */
-class JpaQueryParsingToken implements QueryToken {
-
-	/**
-	 * Commonly use tokens.
-	 */
-	static final JpaQueryParsingToken TOKEN_NONE = JpaQueryParsingToken.token("");
-	static final JpaQueryParsingToken TOKEN_COMMA = JpaQueryParsingToken.token(", ");
-	static final JpaQueryParsingToken TOKEN_SPACE = JpaQueryParsingToken.token(" ");
-	static final JpaQueryParsingToken TOKEN_DOT = JpaQueryParsingToken.token(".");
-	static final JpaQueryParsingToken TOKEN_EQUALS = JpaQueryParsingToken.token(" = ");
-	static final JpaQueryParsingToken TOKEN_OPEN_PAREN = JpaQueryParsingToken.token("(");
-	static final JpaQueryParsingToken TOKEN_CLOSE_PAREN = JpaQueryParsingToken.token(")");
-	static final JpaQueryParsingToken TOKEN_ORDER_BY = JpaQueryParsingToken.expression("order by");
-	static final JpaQueryParsingToken TOKEN_LOWER_FUNC = JpaQueryParsingToken.token("lower(");
-	static final JpaQueryParsingToken TOKEN_SELECT_COUNT = JpaQueryParsingToken.token("select count(");
-	static final JpaQueryParsingToken TOKEN_COUNT_FUNC = JpaQueryParsingToken.token("count(");
-	static final JpaQueryParsingToken TOKEN_DOUBLE_PIPE = JpaQueryParsingToken.token(" || ");
-	static final JpaQueryParsingToken TOKEN_OPEN_SQUARE_BRACKET = JpaQueryParsingToken.token("[");
-	static final JpaQueryParsingToken TOKEN_CLOSE_SQUARE_BRACKET = JpaQueryParsingToken.token("]");
-	static final JpaQueryParsingToken TOKEN_COLON = JpaQueryParsingToken.token(":");
-	static final JpaQueryParsingToken TOKEN_QUESTION_MARK = JpaQueryParsingToken.token("?");
-	static final JpaQueryParsingToken TOKEN_OPEN_BRACE = JpaQueryParsingToken.token("{");
-	static final JpaQueryParsingToken TOKEN_CLOSE_BRACE = JpaQueryParsingToken.token("}");
-	static final JpaQueryParsingToken TOKEN_DOUBLE_UNDERSCORE = JpaQueryParsingToken.token("__");
-	static final JpaQueryParsingToken TOKEN_AS = JpaQueryParsingToken.expression("AS");
-	static final JpaQueryParsingToken TOKEN_DESC = JpaQueryParsingToken.expression("desc");
-	static final JpaQueryParsingToken TOKEN_ASC = JpaQueryParsingToken.expression("asc");
-	static final JpaQueryParsingToken TOKEN_WITH = JpaQueryParsingToken.expression("WITH");
-	static final JpaQueryParsingToken TOKEN_NOT = JpaQueryParsingToken.expression("NOT");
-	static final JpaQueryParsingToken TOKEN_MATERIALIZED = JpaQueryParsingToken.expression("materialized");
-	static final JpaQueryParsingToken TOKEN_NULLS = JpaQueryParsingToken.expression("NULLS");
-	static final JpaQueryParsingToken TOKEN_FIRST = JpaQueryParsingToken.expression("FIRST");
-	static final JpaQueryParsingToken TOKEN_LAST = JpaQueryParsingToken.expression("LAST");
-
-	/**
-	 * The text value of the token.
-	 */
-	private final String token;
-
-	JpaQueryParsingToken(String token) {
-		this.token = token;
-	}
-
-	static JpaQueryParsingToken token(TerminalNode node) {
-		return token(node.getText());
-	}
-
-	static JpaQueryParsingToken token(Token token) {
-		return token(token.getText());
-	}
-
-	static JpaQueryParsingToken token(String token) {
-		return new JpaQueryParsingToken(token);
-	}
-
-	static JpaQueryParsingToken expression(String expression) {
-		return new JpaExpressionToken(expression);
-	}
-
-	static JpaQueryParsingToken expression(Token token) {
-		return expression(token.getText());
-	}
-
-	static JpaQueryParsingToken expression(TerminalNode node) {
-		return expression(node.getText());
-	}
-
-	static JpaQueryParsingToken ventilated(Token op) {
-		return new JpaQueryParsingToken(" " + op.getText() + " ");
-	}
-
-	String getToken() {
-		return value();
-	}
-
-	public String value() {
-		return token;
-	}
-
-	/**
-	 * Compare whether the given {@link JpaQueryParsingToken token} is equal to the one held by this instance.
-	 *
-	 * @param token must not be {@literal null}.
-	 * @return {@literal true} if both tokens are equals (using case-insensitive comparison).
-	 */
-	public boolean isA(QueryToken token) {
-		return token.value().equalsIgnoreCase(this.value());
-	}
-
-	@Override
-	public String toString() {
-		return getToken();
-	}
-
-	static class JpaExpressionToken extends JpaQueryParsingToken {
-
-		JpaExpressionToken(String token) {
-			super(token);
-		}
-
-		public boolean isExpression() {
-			return true;
-		}
-	}
-}

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/JpaQueryTransformerSupport.java

@@ -1,6 +1,6 @@
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -53,9 +53,9 @@ class JpaQueryTransformerSupport {
 	 * @param sort
 	 * @return
 	 */
-	List<JpaQueryParsingToken> orderBy(String primaryFromAlias, Sort sort) {
+	List<QueryToken> orderBy(String primaryFromAlias, Sort sort) {
 
-		List<JpaQueryParsingToken> tokens = new ArrayList<>();
+		List<QueryToken> tokens = new ArrayList<>();
 
 		sort.forEach(order -> {
 
@@ -64,7 +64,7 @@ class JpaQueryTransformerSupport {
 			StringBuilder builder = new StringBuilder();
 
 			if (order.isIgnoreCase()) {
-				builder.append(TOKEN_LOWER_FUNC.getToken());
+				builder.append(TOKEN_LOWER_FUNC.value());
 			}
 
 			builder.append(generateOrderByArgument(primaryFromAlias, order));
@@ -80,7 +80,7 @@ class JpaQueryTransformerSupport {
 				tokens.add(TOKEN_COMMA);
 			}
 
-			tokens.add(JpaQueryParsingToken.token(builder.toString()));
+			tokens.add(QueryTokens.token(builder.toString()));
 		});
 
 		return tokens;

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/JpqlCountQueryTransformer.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import org.springframework.data.jpa.repository.query.QueryRenderer.QueryRendererBuilder;
 import org.springframework.data.jpa.repository.query.QueryTransformers.CountSelectionTokenStream;
@@ -66,24 +66,24 @@ class JpqlCountQueryTransformer extends JpqlQueryRenderer {
 
 		QueryRendererBuilder builder = QueryRenderer.builder();
 
-		builder.append(JpaQueryParsingToken.expression(ctx.SELECT()));
+		builder.append(QueryTokens.expression(ctx.SELECT()));
 		builder.append(TOKEN_COUNT_FUNC);
 
 		if (countProjection != null) {
-			builder.append(JpaQueryParsingToken.token(countProjection));
+			builder.append(QueryTokens.token(countProjection));
 		}
 
 		QueryRendererBuilder nested = QueryRenderer.builder();
 
 		if (ctx.DISTINCT() != null) {
-			nested.append(JpaQueryParsingToken.expression(ctx.DISTINCT()));
+			nested.append(QueryTokens.expression(ctx.DISTINCT()));
 		}
 
 		if (countProjection == null) {
 
 			if (ctx.DISTINCT() != null) {
 
-				QueryRendererBuilder selectionListbuilder = QueryRendererBuilder.concat(ctx.select_item(), this::visit,
+				QueryTokenStream selectionListbuilder = QueryTokenStream.concat(ctx.select_item(), this::visit,
 						TOKEN_COMMA);
 
 				CountSelectionTokenStream countSelection = QueryTransformers
@@ -91,13 +91,13 @@ class JpqlCountQueryTransformer extends JpqlQueryRenderer {
 
 				if (countSelection.requiresPrimaryAlias()) {
 					// constructor
-					nested.append(new JpaQueryParsingToken(primaryFromAlias));
+					nested.append(QueryTokens.token(primaryFromAlias));
 				} else {
 					// keep all the select items to distinct against
 					nested.append(countSelection);
 				}
 			} else {
-				nested.append(new JpaQueryParsingToken(primaryFromAlias));
+				nested.append(QueryTokens.token(primaryFromAlias));
 			}
 		}
 

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/JpqlQueryIntrospector.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -34,7 +34,7 @@ class JpqlQueryIntrospector extends JpqlBaseVisitor<Void> implements ParsedQuery
 	private final JpqlQueryRenderer renderer = new JpqlQueryRenderer();
 
 	private @Nullable String primaryFromAlias = null;
-	private @Nullable List<JpaQueryParsingToken> projection;
+	private @Nullable List<QueryToken> projection;
 	private boolean projectionProcessed;
 	private boolean hasConstructorExpression = false;
 
@@ -43,7 +43,7 @@ class JpqlQueryIntrospector extends JpqlBaseVisitor<Void> implements ParsedQuery
 		return primaryFromAlias;
 	}
 
-	public List<JpaQueryParsingToken> getProjection() {
+	public List<QueryToken> getProjection() {
 		return projection == null ? Collections.emptyList() : projection;
 	}
 
@@ -66,7 +66,7 @@ class JpqlQueryIntrospector extends JpqlBaseVisitor<Void> implements ParsedQuery
 	public Void visitSelect_clause(JpqlParser.Select_clauseContext ctx) {
 
 		List<JpqlParser.Select_itemContext> selections = ctx.select_item();
-		List<JpaQueryParsingToken> selectItemTokens = new ArrayList<>(selections.size() * 2);
+		List<QueryToken> selectItemTokens = new ArrayList<>(selections.size() * 2);
 
 		for (JpqlParser.Select_itemContext selection : selections) {
 
@@ -74,7 +74,7 @@ class JpqlQueryIntrospector extends JpqlBaseVisitor<Void> implements ParsedQuery
 				selectItemTokens.add(TOKEN_COMMA);
 			}
 
-			selectItemTokens.add(JpaQueryParsingToken.token(renderer.visitSelect_item(selection).build().render()));
+			selectItemTokens.add(QueryTokens.token(QueryRenderer.from(renderer.visitSelect_item(selection)).render()));
 		}
 
 		if (!projectionProcessed) {

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/JpqlSortedQueryTransformer.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import java.util.List;
 
@@ -48,7 +48,7 @@ class JpqlSortedQueryTransformer extends JpqlQueryRenderer {
 	}
 
 	@Override
-	public QueryRendererBuilder visitSelect_statement(JpqlParser.Select_statementContext ctx) {
+	public QueryTokenStream visitSelect_statement(JpqlParser.Select_statementContext ctx) {
 
 		QueryRendererBuilder builder = QueryRenderer.builder();
 
@@ -75,7 +75,7 @@ class JpqlSortedQueryTransformer extends JpqlQueryRenderer {
 	private void doVisitOrderBy(QueryRendererBuilder builder, JpqlParser.Select_statementContext ctx) {
 
 		if (ctx.orderby_clause() != null) {
-			QueryRendererBuilder existingOrder = visit(ctx.orderby_clause());
+			QueryTokenStream existingOrder = visit(ctx.orderby_clause());
 			if (sort.isSorted()) {
 				builder.appendInline(existingOrder);
 			} else {
@@ -85,7 +85,7 @@ class JpqlSortedQueryTransformer extends JpqlQueryRenderer {
 
 		if (sort.isSorted()) {
 
-			List<JpaQueryParsingToken> sortBy = transformerSupport.orderBy(primaryFromAlias, sort);
+			List<QueryToken> sortBy = transformerSupport.orderBy(primaryFromAlias, sort);
 
 			if (ctx.orderby_clause() != null) {
 
@@ -100,24 +100,26 @@ class JpqlSortedQueryTransformer extends JpqlQueryRenderer {
 	}
 
 	@Override
-	public QueryRendererBuilder visitSelect_item(JpqlParser.Select_itemContext ctx) {
+	public QueryTokenStream visitSelect_item(JpqlParser.Select_itemContext ctx) {
 
-		QueryRendererBuilder builder = super.visitSelect_item(ctx);
+		QueryTokenStream tokens = super.visitSelect_item(ctx);
 
-		if (ctx.result_variable() != null) {
-			transformerSupport.registerAlias(builder.lastToken());
+		if (ctx.result_variable() != null && !tokens.isEmpty()) {
+			transformerSupport.registerAlias(tokens.getLast());
 		}
 
-		return builder;
+		return tokens;
 	}
 
 	@Override
-	public QueryRendererBuilder visitJoin(JpqlParser.JoinContext ctx) {
+	public QueryTokenStream visitJoin(JpqlParser.JoinContext ctx) {
 
-		QueryRendererBuilder builder = super.visitJoin(ctx);
+		QueryTokenStream tokens = super.visitJoin(ctx);
 
-		transformerSupport.registerAlias(builder.lastToken());
+		if (!tokens.isEmpty()) {
+			transformerSupport.registerAlias(tokens.getLast());
+		}
 
-		return builder;
+		return tokens;
 	}
 }

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/ParsedQueryIntrospector.java

@@ -29,6 +29,12 @@ import org.springframework.lang.Nullable;
  */
 interface ParsedQueryIntrospector {
 
+	/**
+	 * Visit the parsed tree to introspect the AST tree.
+	 *
+	 * @param tree
+	 * @return
+	 */
 	Void visit(ParseTree tree);
 
 	/**
@@ -39,7 +45,13 @@ interface ParsedQueryIntrospector {
 	@Nullable
 	String getAlias();
 
-	List<JpaQueryParsingToken> getProjection();
+	/**
+	 * @return the primary selection.
+	 */
+	List<QueryToken> getProjection();
 
+	/**
+	 * @return {@code true} if the query uses a constructor expression.
+	 */
 	boolean hasConstructorExpression();
 }

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/QueryEnhancer.java

@@ -62,7 +62,7 @@ public interface QueryEnhancer {
 	/**
 	 * Gets the query we want to use for enhancements.
 	 *
-	 * @return non-null {@link DeclaredQuery} that wraps the query
+	 * @return non-null {@link DeclaredQuery} that wraps the query.
 	 */
 	@Deprecated(forRemoval = true)
 	DeclaredQuery getQuery();

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/QueryRenderer.java

@@ -23,7 +23,6 @@ import java.util.List;
 import java.util.function.Function;
 import java.util.stream.Stream;
 
-import org.antlr.v4.runtime.TokenStream;
 import org.springframework.util.CompositeIterator;
 
 /**
@@ -31,8 +30,7 @@ import org.springframework.util.CompositeIterator;
  * <p>
  * Query rendering consists of multiple building blocks:
  * <ul>
- * <li>{@link JpaQueryParsingToken tokens} and
- * {@link org.springframework.data.jpa.repository.query.JpaQueryParsingToken.JpaExpressionToken expression tokens}</li>
+ * <li>{@link QueryTokens.SimpleQueryToken tokens} and {@link QueryTokens.ExpressionToken expression tokens}</li>
  * <li>{@link QueryRenderer compositions} such as a composition of multiple tokens.</li>
  * <li>{@link QueryRenderer expressions} that are individual parts such as {@code SELECT} or {@code ORDER BY …}</li>
  * <li>{@link QueryRenderer inline expressions} such as composition of tokens and expressions such as function calls
@@ -41,10 +39,20 @@ import org.springframework.util.CompositeIterator;
  *
  * @author Mark Paluch
  */
-abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
+abstract class QueryRenderer implements QueryTokenStream {
 
 	/**
-	 * Creates a QueryRenderer from a collection of {@link JpaQueryParsingToken}.
+	 * Creates a QueryRenderer from a {@link QueryToken}.
+	 *
+	 * @param token
+	 * @return
+	 */
+	static QueryRenderer from(QueryToken token) {
+		return QueryRenderer.from(Collections.singletonList(token));
+	}
+
+	/**
+	 * Creates a QueryRenderer from a collection of {@link QueryToken}.
 	 *
 	 * @param tokens
 	 * @return
@@ -55,6 +63,25 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		return new TokenRenderer(tokensToUse);
 	}
 
+	/**
+	 * Creates a QueryRenderer from a {@link QueryTokenStream}.
+	 *
+	 * @param tokens
+	 * @return
+	 */
+	static QueryRenderer from(QueryTokenStream tokens) {
+
+		if (tokens instanceof QueryRendererBuilder builder) {
+			tokens = builder.current;
+		}
+
+		if (tokens instanceof QueryRenderer renderer) {
+			return renderer;
+		}
+
+		return new QueryStreamRenderer(tokens);
+	}
+
 	/**
 	 * Creates a new empty {@link QueryRenderer}.
 	 *
@@ -79,20 +106,72 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 	abstract String render();
 
 	/**
-	 * Append a {@link QueryRenderer} to create a composed renderer.
-	 *
-	 * @param renderer
-	 * @return
+	 * @return the rendered query.
 	 */
-	QueryRenderer append(QueryRenderer renderer) {
-		return CompositeRenderer.combine(this, renderer);
+	static String render(Iterable<QueryToken> tokenStream) {
+
+		if (tokenStream instanceof QueryRendererBuilder qrb) {
+			tokenStream = qrb.current;
+		}
+
+		if (tokenStream instanceof QueryRenderer qr) {
+			return qr.render();
+		}
+
+		StringBuilder results = null;
+		boolean previousExpression = false;
+
+		Iterator<QueryToken> iterator = tokenStream.iterator();
+		while (iterator.hasNext()) {
+			QueryToken token = iterator.next();
+
+			if (results == null) {
+				if (iterator.hasNext()) {
+					results = new StringBuilder();
+				} else {
+					return token.value();
+				}
+			}
+
+			if (previousExpression) {
+				if (!results.isEmpty() && results.charAt(results.length() - 1) != ' ') {
+					results.append(' ');
+				}
+			}
+
+			previousExpression = token.isExpression();
+			results.append(token.value());
+		}
+
+		return results.toString();
 	}
 
 	/**
-	 * @return {@code true} if the query renderer represents an expression.
+	 * Append a {@link QueryRenderer} to create a composed renderer.
+	 *
+	 * @param tokens
+	 * @return
 	 */
-	public boolean isExpression() {
-		return false;
+	QueryRenderer append(QueryTokenStream tokens) {
+
+		if (tokens instanceof QueryRendererBuilder builder) {
+			tokens = builder.current;
+		}
+
+		if (tokens instanceof QueryRenderer qr) {
+
+			if (isEmpty()) {
+				return qr;
+			}
+
+			return CompositeRenderer.combine(this, qr);
+		}
+
+		if (isEmpty()) {
+			return QueryRenderer.from(tokens);
+		}
+
+		return CompositeRenderer.combine(this, QueryRenderer.from(tokens));
 	}
 
 	@Override
@@ -100,13 +179,46 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		return render();
 	}
 
+	public static QueryRenderer expression(QueryTokenStream tokenStream) {
+
+		if (tokenStream instanceof QueryRendererBuilder builder) {
+			tokenStream = builder.current;
+		}
+
+		if (tokenStream.isEmpty()) {
+			return EmptyQueryRenderer.INSTANCE;
+		}
+
+		if (tokenStream.isExpression()) {
+			return (QueryRenderer) tokenStream;
+		}
+
+		return new ExpressionRenderer((QueryRenderer) tokenStream);
+	}
+
+	public static QueryRenderer inline(QueryTokenStream tokenStream) {
+
+		if (tokenStream instanceof QueryRendererBuilder builder) {
+			tokenStream = builder.current;
+		}
+
+		if (tokenStream.isEmpty()) {
+			return EmptyQueryRenderer.INSTANCE;
+		}
+
+		if (!tokenStream.isExpression()) {
+			return (QueryRenderer) tokenStream;
+		}
+
+		return new InlineRenderer((QueryRenderer) tokenStream);
+	}
+
 	/**
 	 * Composed renderer consisting of one or more QueryRenderers.
 	 */
 	static class CompositeRenderer extends QueryRenderer {
 
 		private final List<QueryRenderer> nested;
-		private int size;
 
 		static CompositeRenderer combine(QueryRenderer root, QueryRenderer nested) {
 
@@ -114,66 +226,124 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 			queryRenderers.add(root);
 			queryRenderers.add(nested);
 
-			return new CompositeRenderer(queryRenderers, root.estimatedSize() + nested.estimatedSize());
+			return new CompositeRenderer(queryRenderers);
 		}
 
-		private CompositeRenderer(List<QueryRenderer> nested, int size) {
-
+		private CompositeRenderer(List<QueryRenderer> nested) {
 			this.nested = nested;
-			this.size = size;
 		}
 
 		@Override
 		String render() {
 
-			StringBuilder builder = new StringBuilder(2048);
+			StringBuilder builder = new StringBuilder(64);
+			String lastAppended = null;
 
 			boolean lastExpression = false;
 			for (QueryRenderer queryRenderer : nested) {
 
-				if (!builder.isEmpty() && builder.charAt(builder.length() - 1) != ' '
-						&& (lastExpression || queryRenderer.isExpression())) {
+				if (lastAppended != null && (lastExpression || queryRenderer.isExpression()) && !builder.isEmpty()
+						&& !lastAppended.endsWith(" ")) {
 					builder.append(' ');
 				}
 
-				builder.append(queryRenderer.render());
+				lastAppended = queryRenderer.render();
+				builder.append(lastAppended);
 				lastExpression = queryRenderer.isExpression();
 			}
 
 			return builder.toString();
 		}
 
-		@Override
-		QueryRenderer append(QueryRenderer renderer) {
+		/**
+		 * Append a {@link QueryRenderer} to create a composed renderer.
+		 *
+		 * @param tokens
+		 * @return
+		 */
+		QueryRenderer append(QueryTokenStream tokens) {
 
-			nested.add(renderer);
-			this.size += renderer.estimatedSize();
-			return this;
+			if (tokens instanceof QueryRendererBuilder builder) {
+				tokens = builder.current;
+			}
+
+			if (tokens instanceof QueryRenderer qr) {
+
+				if (isEmpty()) {
+					return this;
+				}
+
+				if (qr.isEmpty()) {
+					return qr;
+				}
+
+				if (tokens instanceof CompositeRenderer cr) {
+					this.nested.addAll(cr.nested);
+
+					return this;
+				}
+
+			}
+
+			return super.append(tokens);
 		}
 
 		@Override
-		public boolean isExpression() {
-			return !nested.isEmpty() && nested.get(nested.size() - 1).isExpression();
+		public QueryToken getLast() {
+
+			for (int i = nested.size() - 1; i > -1; i--) {
+
+				QueryRenderer renderer = nested.get(i);
+
+				if (!renderer.isEmpty()) {
+					return renderer.getLast();
+				}
+			}
+
+			return null;
 		}
 
 		@Override
 		public Iterator<QueryToken> iterator() {
 
 			CompositeIterator<QueryToken> iterator = new CompositeIterator<>();
-			for (QueryRenderer renderer : nested) {
-				iterator.add(renderer.iterator());
+			for (QueryTokenStream stream : nested) {
+				iterator.add(stream.iterator());
 			}
 			return iterator;
 		}
 
 		@Override
-		public int estimatedSize() {
+		public boolean isEmpty() {
+
+			for (QueryRenderer renderer : nested) {
+				if (!renderer.isEmpty()) {
+					return false;
+				}
+			}
+
+			return true;
+		}
+
+		@Override
+		public int size() {
+
+			int size = 0;
+
+			for (QueryTokenStream stream : nested) {
+				size += stream.size();
+			}
 			return size;
 		}
+
+		@Override
+		public boolean isExpression() {
+			return !nested.isEmpty() && nested.get(nested.size() - 1).isExpression();
+		}
 	}
 
 	/**
-	 * Renderer using {@link JpaQueryParsingToken}.
+	 * Renderer using {@link QueryTokens.SimpleQueryToken}.
 	 */
 	static class TokenRenderer extends QueryRenderer {
 
@@ -189,19 +359,14 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		@Override
-		QueryRenderer append(QueryRenderer renderer) {
+		QueryRenderer append(QueryTokenStream tokens) {
 
-			if (renderer instanceof TokenRenderer tr) {
+			if (tokens instanceof TokenRenderer tr) {
 				this.tokens.addAll(tr.tokens);
 				return this;
 			}
 
-			return super.append(renderer);
-		}
-
-		@Override
-		public boolean isExpression() {
-			return !tokens.isEmpty() && tokens.get(tokens.size() - 1).isExpression();
+			return super.append(tokens);
 		}
 
 		@Override
@@ -220,12 +385,32 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		@Override
-		public int estimatedSize() {
+		public QueryToken getFirst() {
+			return tokens.isEmpty() ? null : tokens.get(0);
+		}
+
+		@Override
+		public QueryToken getLast() {
+			return tokens.isEmpty() ? null : tokens.get(tokens.size() - 1);
+		}
+
+		@Override
+		public int size() {
 			return tokens.size();
 		}
 
+		@Override
+		public boolean isEmpty() {
+			return tokens.isEmpty();
+		}
+
+		@Override
+		public boolean isExpression() {
+			return !tokens.isEmpty() && getLast().isExpression();
+		}
+
 		/**
-		 * Render a list of {@link JpaQueryParsingToken}s into a string.
+		 * Render a list of {@link QueryTokens.SimpleQueryToken}s into a string.
 		 *
 		 * @param tokens
 		 * @return rendered string containing either a query or some subset of that query
@@ -236,45 +421,67 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 				return render(tpr);
 			}
 
-			if(tokens instanceof QueryRendererBuilder qrb) {
+			if (tokens instanceof QueryRendererBuilder qrb) {
 				return qrb.build().render();
 			}
 
-			if(tokens instanceof QueryRenderer qr) {
+			if (tokens instanceof QueryRenderer qr) {
 				return qr.render();
 			}
 
 			throw new IllegalArgumentException("Unknown token type %s".formatted(tokens));
 		}
 
+	}
 
-		static String render(Collection<QueryToken> tokens) {
+	static class QueryStreamRenderer extends QueryRenderer {
 
-			StringBuilder results = new StringBuilder();
+		private final QueryTokenStream tokens;
 
-			boolean previousExpression = false;
-
-			for (QueryToken jpaQueryParsingToken : tokens) {
-
-				if (previousExpression) {
-					if (!results.isEmpty() && results.charAt(results.length() - 1) != ' ') {
-						results.append(' ');
-					}
-				}
-
-				previousExpression = jpaQueryParsingToken.isExpression();
-				results.append(jpaQueryParsingToken.value());
-			}
-
-			return results.toString();
+		public QueryStreamRenderer(QueryTokenStream tokens) {
+			this.tokens = tokens;
 		}
 
+		@Override
+		String render() {
+			return render(tokens);
+		}
+
+		@Override
+		public Iterator<QueryToken> iterator() {
+			return tokens.iterator();
+		}
+
+		@Override
+		public QueryToken getFirst() {
+			return tokens.getFirst();
+		}
+
+		@Override
+		public QueryToken getLast() {
+			return tokens.getLast();
+		}
+
+		@Override
+		public int size() {
+			return tokens.size();
+		}
+
+		@Override
+		public boolean isEmpty() {
+			return tokens.isEmpty();
+		}
+
+		@Override
+		public boolean isExpression() {
+			return !tokens.isEmpty() && tokens.getLast().isExpression();
+		}
 	}
 
 	/**
 	 * Builder for {@link QueryRenderer}.
 	 */
-	static class QueryRendererBuilder implements QueryTokenStream<QueryToken> {
+	static class QueryRendererBuilder implements QueryTokenStream {
 
 		protected QueryRenderer current = QueryRenderer.empty();
 
@@ -289,7 +496,7 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		 * @param <T>
 		 */
 		public static <T> QueryRendererBuilder concat(Collection<T> elements, Function<T, QueryRendererBuilder> visitor,
-			QueryToken separator) {
+				QueryToken separator) {
 			return concat(elements, visitor, QueryRendererBuilder::toInline, separator);
 		}
 
@@ -334,7 +541,7 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		/**
-		 * Create and initialize a QueryRendererBuilder from a {@link JpaQueryParsingToken}.
+		 * Create and initialize a QueryRendererBuilder from a {@link QueryTokens.SimpleQueryToken}.
 		 *
 		 * @param token
 		 * @return
@@ -344,39 +551,25 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		/**
-		 * Append a {@link JpaQueryParsingToken}.
+		 * Append a {@link QueryTokens.SimpleQueryToken}.
 		 *
 		 * @param token
 		 * @return {@code this} builder.
 		 */
 		QueryRendererBuilder append(QueryToken token) {
-			return append(List.of(token));
+			return append(QueryRenderer.from(token));
 		}
 
 		/**
-		 * Append a collection of {@link JpaQueryParsingToken}.
+		 * Append a collection of {@link QueryTokens.SimpleQueryToken}.
 		 *
 		 * @param tokens
 		 * @return {@code this} builder.
 		 */
-		QueryRendererBuilder append(Collection<? extends QueryToken> tokens) {
+		QueryRendererBuilder append(List<? extends QueryToken> tokens) {
 			return append(QueryRenderer.from(tokens));
 		}
 
-		QueryRendererBuilder append(QueryTokenStream<? extends QueryToken> tokens) {
-			return append(QueryRenderer.from(tokens.toList()));
-		}
-
-		/**
-		 * Append a QueryRendererBuilder.
-		 *
-		 * @param builder
-		 * @return {@code this} builder.
-		 */
-		QueryRendererBuilder append(QueryRendererBuilder builder) {
-			return append(builder.current);
-		}
-
 		/**
 		 * Append a QueryRendererBuilder as expression.
 		 *
@@ -387,29 +580,19 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 			return appendExpression(builder.current);
 		}
 
-		/**
-		 * Append a QueryRendererBuilder as inline.
-		 *
-		 * @param builder
-		 * @return {@code this} builder.
-		 */
-		QueryRendererBuilder appendInline(QueryRendererBuilder builder) {
-			return appendInline(builder.current);
-		}
-
 		/**
 		 * Append a QueryRenderer.
 		 *
-		 * @param renderer
+		 * @param stream
 		 * @return {@code this} builder.
 		 */
-		QueryRendererBuilder append(QueryRenderer renderer) {
+		QueryRendererBuilder append(QueryTokenStream stream) {
 
-			if (renderer instanceof EmptyQueryRenderer) {
+			if (stream.isEmpty()) {
 				return this;
 			}
 
-			current = current.append(renderer);
+			current = current.append(stream);
 
 			return this;
 		}
@@ -417,16 +600,16 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		/**
 		 * Append a QueryRenderer inline.
 		 *
-		 * @param renderer
+		 * @param stream
 		 * @return {@code this} builder.
 		 */
-		QueryRendererBuilder appendInline(QueryRenderer renderer) {
+		QueryRendererBuilder appendInline(QueryTokenStream stream) {
 
-			if (renderer instanceof EmptyQueryRenderer) {
+			if (stream.isEmpty()) {
 				return this;
 			}
 
-			current = current.append(!renderer.isExpression() ? renderer : new InlineRenderer(renderer));
+			current = current.append(QueryRenderer.inline(stream));
 
 			return this;
 		}
@@ -434,51 +617,20 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		/**
 		 * Append a QueryRenderer as expression.
 		 *
-		 * @param renderer
+		 * @param tokens
 		 * @return {@code this} builder.
 		 */
-		QueryRendererBuilder appendExpression(QueryRenderer renderer) {
+		QueryRendererBuilder appendExpression(QueryTokenStream tokens) {
 
-			if (renderer instanceof EmptyQueryRenderer) {
+			if (tokens.isEmpty()) {
 				return this;
 			}
 
-			current = current.append(renderer.isExpression() ? renderer : new ExpressionRenderer(renderer));
+			current = current.append(QueryRenderer.expression(tokens));
 
 			return this;
 		}
 
-		/**
-		 * Return whet the builder is empty.
-		 *
-		 * @return
-		 */
-		public boolean isEmpty() {
-			return current.isEmpty();
-		}
-
-		public QueryRenderer build() {
-			return current;
-		}
-
-		@Override
-		public String toString() {
-			return current.render();
-		}
-
-		private QueryRenderer toExpression() {
-
-			if (current instanceof ExpressionRenderer) {
-				return current;
-			}
-
-			return new ExpressionRenderer(current);
-		}
-
-		public QueryRenderer toInline() {
-			return new InlineRenderer(current);
-		}
-
 		@Override
 		public List<QueryToken> toList() {
 			return current.toList();
@@ -490,14 +642,62 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		@Override
-		public int estimatedSize() {
-			return current.estimatedSize();
+		public QueryToken getFirst() {
+			return current.getFirst();
+		}
+
+		@Override
+		public QueryToken getLast() {
+			return current.getLast();
+		}
+
+		@Override
+		public boolean isExpression() {
+			return current.isExpression();
+		}
+
+		/**
+		 * Return whet the builder is empty.
+		 *
+		 * @return
+		 */
+		@Override
+		public boolean isEmpty() {
+			return current.isEmpty();
+		}
+
+		@Override
+		public int size() {
+			return current.size();
 		}
 
 		@Override
 		public Iterator<QueryToken> iterator() {
 			return current.iterator();
 		}
+
+		@Override
+		public String toString() {
+			return current.render();
+		}
+
+		public QueryRenderer build() {
+			return current;
+		}
+
+		QueryRenderer toExpression() {
+
+			if (current instanceof ExpressionRenderer) {
+				return current;
+			}
+
+			return QueryRenderer.expression(current);
+		}
+
+		public QueryRenderer toInline() {
+			return new InlineRenderer(current);
+		}
+
 	}
 
 	private static class InlineRenderer extends QueryRenderer {
@@ -529,8 +729,28 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		@Override
-		public int estimatedSize() {
-			return delegate.estimatedSize();
+		public QueryToken getFirst() {
+			return delegate.getFirst();
+		}
+
+		@Override
+		public QueryToken getLast() {
+			return delegate.getLast();
+		}
+
+		@Override
+		public boolean isEmpty() {
+			return delegate.isEmpty();
+		}
+
+		@Override
+		public int size() {
+			return delegate.size();
+		}
+
+		@Override
+		public boolean isExpression() {
+			return false;
 		}
 	}
 
@@ -547,11 +767,6 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 			return delegate.render();
 		}
 
-		@Override
-		public boolean isExpression() {
-			return true;
-		}
-
 		@Override
 		public Stream<QueryToken> stream() {
 			return delegate.stream();
@@ -568,9 +783,30 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		@Override
-		public int estimatedSize() {
-			return delegate.estimatedSize();
+		public QueryToken getFirst() {
+			return delegate.getFirst();
 		}
+
+		@Override
+		public QueryToken getLast() {
+			return delegate.getLast();
+		}
+
+		@Override
+		public boolean isEmpty() {
+			return delegate.isEmpty();
+		}
+
+		@Override
+		public int size() {
+			return delegate.size();
+		}
+
+		@Override
+		public boolean isExpression() {
+			return true;
+		}
+
 	}
 
 	private static class EmptyQueryRenderer extends QueryRenderer {
@@ -583,13 +819,17 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		@Override
-		QueryRenderer append(QueryRenderer renderer) {
-			return renderer;
-		}
+		QueryRenderer append(QueryTokenStream tokens) {
 
-		@Override
-		public boolean isEmpty() {
-			return true;
+			if (tokens.isEmpty()) {
+				return this;
+			}
+
+			if (tokens instanceof QueryRenderer qr) {
+				return qr;
+			}
+
+			return QueryRenderer.from(tokens);
 		}
 
 		@Override
@@ -608,8 +848,18 @@ abstract class QueryRenderer implements QueryTokenStream<QueryToken> {
 		}
 
 		@Override
-		public int estimatedSize() {
+		public boolean isEmpty() {
+			return true;
+		}
+
+		@Override
+		public int size() {
 			return 0;
 		}
+
+		@Override
+		public boolean isExpression() {
+			return false;
+		}
 	}
 }

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/QueryToken.java

@@ -16,15 +16,24 @@
 package org.springframework.data.jpa.repository.query;
 
 /**
+ * Interface defining a token. Tokens are atomic elements from which queries are built. Tokens can be inline tokens that
+ * do not require spacing or expressions that must be separated by spaces, commas, etc.
+ *
  * @author Christoph Strobl
+ * @since 3.4
  */
-public interface QueryToken {
+interface QueryToken {
 
+	/**
+	 * @return the token value (i.e. its content).
+	 */
 	String value();
 
+	/**
+	 * @return {@code true} if the token represents an expression.
+	 */
 	default boolean isExpression() {
 		return false;
 	}
 
-	boolean isA(QueryToken queryToken);
 }

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/QueryTokenStream.java

@@ -15,24 +15,134 @@
  */
 package org.springframework.data.jpa.repository.query;
 
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.function.Function;
+
 import org.springframework.data.util.Streamable;
 import org.springframework.lang.Nullable;
 import org.springframework.util.CollectionUtils;
 
 /**
+ * Stream of {@link QueryToken}.
+ *
  * @author Christoph Strobl
+ * @author Mark Paluch
+ * @since 3.4
  */
-public interface QueryTokenStream<T extends QueryToken> extends Streamable<T> {
+interface QueryTokenStream extends Streamable<QueryToken> {
 
-	@Nullable
-	default T firstToken() {
-		return CollectionUtils.firstElement(toList());
+	/**
+	 * Creates an empty stream.
+	 *
+	 * @return
+	 */
+	static QueryTokenStream empty() {
+		return EmptyQueryTokenStream.INSTANCE;
 	}
 
+	/**
+	 * Compose a {@link QueryTokenStream} from a collection of inline elements.
+	 *
+	 * @param elements collection of elements.
+	 * @param visitor visitor function converting the element into a {@link QueryTokenStream}.
+	 * @param separator separator token.
+	 * @return the composed token stream.
+	 */
+	static <T> QueryTokenStream concat(Collection<T> elements, Function<T, QueryTokenStream> visitor,
+			QueryToken separator) {
+		return concat(elements, visitor, QueryRenderer::inline, separator);
+	}
+
+	/**
+	 * Compose a {@link QueryTokenStream} from a collection of expression elements.
+	 *
+	 * @param elements collection of elements.
+	 * @param visitor visitor function converting the element into a {@link QueryTokenStream}.
+	 * @param separator separator token.
+	 * @return the composed token stream.
+	 */
+	static <T> QueryTokenStream concatExpressions(Collection<T> elements, Function<T, QueryTokenStream> visitor,
+			QueryToken separator) {
+		return concat(elements, visitor, QueryRenderer::expression, separator);
+	}
+
+	/**
+	 * Compose a {@link QueryTokenStream} from a collection of elements.
+	 *
+	 * @param elements collection of elements.
+	 * @param visitor visitor function converting the element into a {@link QueryTokenStream}.
+	 * @param separator separator token.
+	 * @param postProcess post-processing function to map {@link QueryTokenStream}.
+	 * @return the composed token stream.
+	 */
+	static <T> QueryTokenStream concat(Collection<T> elements, Function<T, QueryTokenStream> visitor,
+			Function<QueryTokenStream, QueryTokenStream> postProcess, QueryToken separator) {
+
+		QueryRenderer.QueryRendererBuilder builder = null;
+		QueryTokenStream firstElement = null;
+		for (T element : elements) {
+
+			QueryTokenStream tokenStream = postProcess.apply(visitor.apply(element));
+
+			if (firstElement == null) {
+				firstElement = tokenStream;
+				continue;
+			}
+
+			if (builder == null) {
+				builder = QueryRenderer.builder();
+				builder.append(firstElement);
+			}
+
+			if (!builder.isEmpty()) {
+				builder.append(separator);
+			}
+			builder.append(tokenStream);
+		}
+
+		if (builder != null) {
+			return builder;
+		}
+
+		if (firstElement != null) {
+			return firstElement;
+		}
+
+		return QueryTokenStream.empty();
+	}
+
+	/**
+	 * @return the first query token or {@code null} if empty.
+	 */
 	@Nullable
-	default T lastToken() {
+	default QueryToken getFirst() {
+
+		Iterator<QueryToken> it = iterator();
+		return it.hasNext() ? it.next() : null;
+	}
+
+	/**
+	 * @return the last query token or {@code null} if empty.
+	 */
+	@Nullable
+	default QueryToken getLast() {
 		return CollectionUtils.lastElement(toList());
 	}
 
-	int estimatedSize();
+	/**
+	 * @return {@code true} if this stream represents a query expression.
+	 */
+	boolean isExpression();
+
+	/**
+	 * @return the number of tokens.
+	 */
+	int size();
+
+	/**
+	 * @return {@code true} if this stream contains no tokens.
+	 */
+	boolean isEmpty();
+
 }

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/QueryTokens.java

@@ -0,0 +1,190 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.jpa.repository.query;
+
+import java.util.function.Supplier;
+
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.tree.TerminalNode;
+
+/**
+ * Utility class to create query tokens.
+ *
+ * @author Mark Paluch
+ * @since 3.4
+ */
+class QueryTokens {
+
+	/**
+	 * Commonly use tokens.
+	 */
+	static final QueryToken TOKEN_NONE = token("");
+	static final QueryToken TOKEN_COMMA = token(", ");
+	static final QueryToken TOKEN_SPACE = token(" ");
+	static final QueryToken TOKEN_DOT = token(".");
+	static final QueryToken TOKEN_EQUALS = token(" = ");
+	static final QueryToken TOKEN_OPEN_PAREN = token("(");
+	static final QueryToken TOKEN_CLOSE_PAREN = token(")");
+	static final QueryToken TOKEN_ORDER_BY = expression("order by");
+	static final QueryToken TOKEN_LOWER_FUNC = token("lower(");
+	static final QueryToken TOKEN_SELECT_COUNT = token("select count(");
+	static final QueryToken TOKEN_COUNT_FUNC = token("count(");
+	static final QueryToken TOKEN_DOUBLE_PIPE = token(" || ");
+	static final QueryToken TOKEN_OPEN_SQUARE_BRACKET = token("[");
+	static final QueryToken TOKEN_CLOSE_SQUARE_BRACKET = token("]");
+	static final QueryToken TOKEN_COLON = token(":");
+	static final QueryToken TOKEN_QUESTION_MARK = token("?");
+	static final QueryToken TOKEN_OPEN_BRACE = token("{");
+	static final QueryToken TOKEN_CLOSE_BRACE = token("}");
+	static final QueryToken TOKEN_DOUBLE_UNDERSCORE = token("__");
+	static final QueryToken TOKEN_AS = expression("AS");
+	static final QueryToken TOKEN_DESC = expression("desc");
+	static final QueryToken TOKEN_ASC = expression("asc");
+	static final QueryToken TOKEN_WITH = expression("WITH");
+	static final QueryToken TOKEN_NOT = expression("NOT");
+	static final QueryToken TOKEN_MATERIALIZED = expression("materialized");
+	static final QueryToken TOKEN_NULLS = expression("NULLS");
+	static final QueryToken TOKEN_FIRST = expression("FIRST");
+	static final QueryToken TOKEN_LAST = expression("LAST");
+
+	/**
+	 * Creates a {@link QueryToken token} from an ANTLR {@link TerminalNode}.
+	 *
+	 * @param node
+	 * @return
+	 */
+	static QueryToken token(TerminalNode node) {
+		return token(node.getText());
+	}
+
+	/**
+	 * Creates a {@link QueryToken token} from an ANTLR {@link Token}.
+	 *
+	 * @param token
+	 * @return
+	 */
+	static QueryToken token(Token token) {
+		return token(token.getText());
+	}
+
+	/**
+	 * Creates a {@link QueryToken token} from a string {@code token}.
+	 *
+	 * @param token
+	 * @return
+	 */
+	static QueryToken token(String token) {
+		return new SimpleQueryToken(token);
+	}
+
+	/**
+	 * Creates a ventilated token that is embedded in spaces.
+	 *
+	 * @param token
+	 * @return
+	 */
+	static QueryToken ventilated(Token token) {
+		return new SimpleQueryToken(" " + token.getText() + " ");
+	}
+
+	/**
+	 * Creates a {@link QueryToken expression} from an ANTLR {@link TerminalNode}.
+	 *
+	 * @param node
+	 * @return
+	 */
+	static QueryToken expression(TerminalNode node) {
+		return expression(node.getText());
+	}
+
+	/**
+	 * Creates a {@link QueryToken expression} from an ANTLR {@link Token}.
+	 *
+	 * @param token
+	 * @return
+	 */
+	static QueryToken expression(Token token) {
+		return expression(token.getText());
+	}
+
+	/**
+	 * Creates a {@link QueryToken token} from a string {@code expression}.
+	 *
+	 * @param expression
+	 * @return
+	 */
+	static QueryToken expression(String expression) {
+		return new ExpressionToken(expression);
+	}
+
+	/**
+	 * A value type used to represent a JPA query token. NOTE: Sometimes the token's value is based upon a value found
+	 * later in the parsing process, so the text itself is wrapped in a {@link Supplier}.
+	 *
+	 * @author Greg Turnquist
+	 * @author Christoph Strobl
+	 * @since 3.1
+	 */
+	static class SimpleQueryToken implements QueryToken {
+
+		/**
+		 * The text value of the token.
+		 */
+		private final String token;
+
+		SimpleQueryToken(String token) {
+			this.token = token;
+		}
+
+		public String value() {
+			return token;
+		}
+
+		@Override
+		public final boolean equals(Object object) {
+			if (this == object) {
+				return true;
+			}
+
+			if (!(object instanceof QueryToken that)) {
+				return false;
+			}
+
+			return value().equalsIgnoreCase(that.value());
+		}
+
+		@Override
+		public int hashCode() {
+			return value().hashCode();
+		}
+
+		@Override
+		public String toString() {
+			return value();
+		}
+	}
+
+	static class ExpressionToken extends SimpleQueryToken {
+
+		ExpressionToken(String token) {
+			super(token);
+		}
+
+		public boolean isExpression() {
+			return true;
+		}
+	}
+}

spring-data-jpa/src/main/java/org/springframework/data/jpa/repository/query/QueryTransformers.java

@@ -15,7 +15,7 @@
  */
 package org.springframework.data.jpa.repository.query;
 
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
+import static org.springframework.data.jpa.repository.query.QueryTokens.*;
 
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -29,30 +29,29 @@ import java.util.List;
  */
 class QueryTransformers {
 
-	static CountSelectionTokenStream filterCountSelection(QueryTokenStream<QueryToken> selection) {
+	static CountSelectionTokenStream filterCountSelection(QueryTokenStream selection) {
 
-		List<QueryToken> target = new ArrayList<>(selection.estimatedSize());
+		List<QueryToken> target = new ArrayList<>(selection.size());
 		boolean skipNext = false;
 		boolean containsNew = false;
 
 		for (QueryToken token : selection) {
 
-
 			if (skipNext) {
 				skipNext = false;
 				continue;
 			}
 
-			if (token.isA(TOKEN_AS)) {
+			if (token.equals(TOKEN_AS)) {
 				skipNext = true;
 				continue;
 			}
 
-			if (!token.isA(TOKEN_COMMA) && token.isExpression()) {
-				token = JpaQueryParsingToken.token(token.value());
+			if (!token.equals(TOKEN_COMMA) && token.isExpression()) {
+				token = QueryTokens.token(token.value());
 			}
 
-			if(!containsNew && token.value().contains("new")) {
+			if (!containsNew && token.value().contains("new")) {
 				containsNew = true;
 			}
 
@@ -62,19 +61,14 @@ class QueryTransformers {
 		return new CountSelectionTokenStream(target, containsNew);
 	}
 
-	static class CountSelectionTokenStream implements QueryTokenStream<QueryToken> {
+	static class CountSelectionTokenStream implements QueryTokenStream {
 
 		private final List<QueryToken> tokens;
 		private final boolean requiresPrimaryAlias;
 
-		public CountSelectionTokenStream(List<QueryToken> tokens, boolean containsNew) {
+		public CountSelectionTokenStream(List<QueryToken> tokens, boolean requiresPrimaryAlias) {
 			this.tokens = tokens;
-			this.requiresPrimaryAlias = containsNew;
-		}
-
-		@Override
-		public int estimatedSize() {
-			return tokens.size();
+			this.requiresPrimaryAlias = requiresPrimaryAlias;
 		}
 
 		@Override
@@ -87,9 +81,25 @@ class QueryTransformers {
 			return tokens;
 		}
 
+		@Override
+		public int size() {
+			return tokens.size();
+		}
+
+		@Override
+		public boolean isExpression() {
+			return true;
+		}
+
+		@Override
+		public boolean isEmpty() {
+			return tokens.isEmpty();
+		}
+
 		public boolean requiresPrimaryAlias() {
 			return requiresPrimaryAlias;
 		}
+
 	}
 
 }

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/EqlComplianceTests.java

@@ -16,7 +16,6 @@
 package org.springframework.data.jpa.repository.query;
 
 import static org.assertj.core.api.Assertions.*;
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/EqlQueryRendererTests.java

@@ -16,7 +16,6 @@
 package org.springframework.data.jpa.repository.query;
 
 import static org.assertj.core.api.Assertions.*;
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
 
 import java.util.stream.Stream;
 

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/EqlSpecificationTests.java

@@ -16,7 +16,6 @@
 package org.springframework.data.jpa.repository.query;
 
 import static org.assertj.core.api.Assertions.*;
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/HqlQueryRendererTests.java

@@ -55,7 +55,8 @@ class HqlQueryRendererTests {
 
 		HqlParser.StartContext parsedQuery = parser.start();
 
-		return new HqlQueryRenderer().visit(parsedQuery).build().render();
+		QueryTokenStream tokens = new HqlQueryRenderer().visit(parsedQuery);
+		return QueryRenderer.from(tokens).render();
 	}
 
 	static Stream<Arguments> reservedWords() {

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/HqlQueryTransformerTests.java

@@ -499,7 +499,7 @@ class HqlQueryTransformerTests {
 				select sr from maxId m join SnapshotReference sr on sr.snapshot.id = m.snapshotId
 				""");
 
-		assertThat(countQuery).startsWith("WITH maxId AS(select max(sr.snapshot.id) snapshotId from SnapshotReference sr")
+		assertThat(countQuery).startsWith("WITH maxId AS (select max(sr.snapshot.id) snapshotId from SnapshotReference sr")
 				.endsWith("select count(m) from maxId m join SnapshotReference sr on sr.snapshot.id = m.snapshotId");
 	}
 
@@ -514,7 +514,7 @@ class HqlQueryTransformerTests {
 				""", Sort.by("sr.snapshot"));
 
 		assertThat(sortedQuery).startsWith(
-				"WITH maxId AS(select max(sr.snapshot.id) snapshotId from SnapshotReference sr where sr.id.selectionId = ?1 and sr.enabled group by sr.userId )")
+				"WITH maxId AS (select max(sr.snapshot.id) snapshotId from SnapshotReference sr where sr.id.selectionId = ?1 and sr.enabled group by sr.userId)")
 				.endsWith(
 						"select sr from maxId m join SnapshotReference sr on sr.snapshot.id = m.snapshotId order by sr.snapshot asc");
 	}

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/HqlSpecificationTests.java

@@ -16,7 +16,6 @@
 package org.springframework.data.jpa.repository.query;
 
 import static org.assertj.core.api.Assertions.*;
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/JpqlComplianceTests.java

@@ -16,18 +16,16 @@
 package org.springframework.data.jpa.repository.query;
 
 import static org.assertj.core.api.Assertions.*;
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.junit.jupiter.api.Test;
-import org.springframework.data.jpa.repository.query.QueryRenderer.TokenRenderer;
 
 /**
  * Test to verify compliance of {@link JpqlParser} with standard SQL. Other than {@link JpqlSpecificationTests} tests in
  * this class check that the parser follows a lenient approach and does not error on well known concepts like numeric
  * suffix.
- * 
+ *
  * @author Christoph Strobl
  */
 class JpqlComplianceTests {
@@ -41,7 +39,7 @@ class JpqlComplianceTests {
 
 		JpqlParser.StartContext parsedQuery = parser.start();
 
-		return TokenRenderer.render(new JpqlQueryRenderer().visit(parsedQuery));
+		return QueryRenderer.render(new JpqlQueryRenderer().visit(parsedQuery));
 	}
 
 	private void assertQuery(String query) {
@@ -60,11 +58,11 @@ class JpqlComplianceTests {
 	@Test // GH-3277
 	void numericLiterals() {
 
-		assertQuery("SELECT e FROM  Employee e WHERE e.id = 1234");
-		assertQuery("SELECT e FROM  Employee e WHERE e.id = 1234L");
-		assertQuery("SELECT s FROM  Stat s WHERE s.ratio > 3.14");
-		assertQuery("SELECT s FROM  Stat s WHERE s.ratio > 3.14F");
-		assertQuery("SELECT s FROM  Stat s WHERE s.ratio > 3.14e32D");
+		assertQuery("SELECT e FROM Employee e WHERE e.id = 1234");
+		assertQuery("SELECT e FROM Employee e WHERE e.id = 1234L");
+		assertQuery("SELECT s FROM Stat s WHERE s.ratio > 3.14");
+		assertQuery("SELECT s FROM Stat s WHERE s.ratio > 3.14F");
+		assertQuery("SELECT s FROM Stat s WHERE s.ratio > 3.14e32D");
 	}
 
 	@Test // GH-3308

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/JpqlQueryRendererTests.java

@@ -16,7 +16,6 @@
 package org.springframework.data.jpa.repository.query;
 
 import static org.assertj.core.api.Assertions.*;
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
 
 import java.util.stream.Stream;
 

spring-data-jpa/src/test/java/org/springframework/data/jpa/repository/query/JpqlSpecificationTests.java

@@ -16,7 +16,6 @@
 package org.springframework.data.jpa.repository.query;
 
 import static org.assertj.core.api.Assertions.*;
-import static org.springframework.data.jpa.repository.query.JpaQueryParsingToken.*;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;