Spring/spring-data-jpa
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DATAJPA-912 - Optimize paged query execution.

Browse Source 
We execute paged queries now in an optimized way. The data is obtained for each paged execution but the count query is deferred. We determine the total from the pageable and the results in which we don't hit the page size bounds (i.e. results are less than a full page without offset or results are greater 0 and less than a full page with offset). In all other cases we issue an additional count query.
This commit is contained in:
Mark Paluch
2016-06-29 09:38:49 +02:00
committed by Oliver Gierke
parent a373a28637
commit edf63705c9
7 changed files with 219 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/java/org/springframework/data/jpa/repository/query/JpaQueryExecution.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2008-2015 the original author or authors.
* Copyright 2008-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@
package org.springframework.data.jpa.repository.query;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import javax.persistence.EntityManager;
@@ -27,7 +26,6 @@ import javax.persistence.StoredProcedureQuery;
import org.springframework.core.convert.ConversionService;
import org.springframework.core.convert.support.ConfigurableConversionService;
import org.springframework.core.convert.support.DefaultConversionService;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.domain.SliceImpl;
@@ -35,6 +33,8 @@ import org.springframework.data.jpa.provider.PersistenceProvider;
import org.springframework.data.repository.query.ParameterAccessor;
import org.springframework.data.repository.query.Parameters;
import org.springframework.data.repository.query.ParametersParameterAccessor;
import org.springframework.data.repository.support.PageableExecutionUtils;
import org.springframework.data.repository.support.PageableExecutionUtils.TotalSupplier;
import org.springframework.data.util.CloseableIterator;
import org.springframework.data.util.StreamUtils;
import org.springframework.util.Assert;
@@ -46,6 +46,7 @@ import org.springframework.util.Assert;
*
* @author Oliver Gierke
* @author Thomas Darimont
* @author Mark Paluch
*/
public abstract class JpaQueryExecution {
@@ -171,27 +172,20 @@ public abstract class JpaQueryExecution {
@Override
@SuppressWarnings("unchecked")
protected Object doExecute(AbstractJpaQuery repositoryQuery, Object[] values) {
// Execute query to compute total
Query projection = repositoryQuery.createCountQuery(values);
List<?> totals = projection.getResultList();
Long total = totals.size() == 1 ? CONVERSION_SERVICE.convert(totals.get(0), Long.class) : totals.size();
protected Object doExecute(final AbstractJpaQuery repositoryQuery, final Object[] values) {
ParameterAccessor accessor = new ParametersParameterAccessor(parameters, values);
Pageable pageable = accessor.getPageable();
if (total.equals(0L)) {
return new PageImpl<Object>(Collections.emptyList(), pageable, total);
}
Query query = repositoryQuery.createQuery(values);
List<Object> content = pageable == null || total > pageable.getOffset() ? query.getResultList()
: Collections.emptyList();
return PageableExecutionUtils.getPage(query.getResultList(), accessor.getPageable(), new TotalSupplier() {
return new PageImpl<Object>(content, pageable, total);
@Override
public long get() {
List<?> totals = repositoryQuery.createCountQuery(values).getResultList();
return (totals.size() == 1 ? CONVERSION_SERVICE.convert(totals.get(0), Long.class) : totals.size());
}
});
}
}

18
src/main/java/org/springframework/data/jpa/repository/support/QueryDslJpaRepository.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2008-2015 the original author or authors.
* Copyright 2008-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@
package org.springframework.data.jpa.repository.support;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
@@ -24,13 +23,14 @@ import javax.persistence.EntityManager;
import javax.persistence.LockModeType;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.querydsl.EntityPathResolver;
import org.springframework.data.querydsl.QSort;
import org.springframework.data.querydsl.QueryDslPredicateExecutor;
import org.springframework.data.querydsl.SimpleEntityPathResolver;
import org.springframework.data.repository.support.PageableExecutionUtils;
import org.springframework.data.repository.support.PageableExecutionUtils.TotalSupplier;
import com.querydsl.core.types.EntityPath;
import com.querydsl.core.types.OrderSpecifier;
@@ -45,6 +45,7 @@ import com.querydsl.jpa.impl.AbstractJPAQuery;
*
* @author Oliver Gierke
* @author Thomas Darimont
* @author Mark Paluch
*/
public class QueryDslJpaRepository<T, ID extends Serializable> extends SimpleJpaRepository<T, ID>
implements QueryDslPredicateExecutor<T> {
@@ -136,13 +137,16 @@ public class QueryDslJpaRepository<T, ID extends Serializable> extends SimpleJpa
@Override
public Page<T> findAll(Predicate predicate, Pageable pageable) {
JPQLQuery<?> countQuery = createQuery(predicate);
final JPQLQuery<?> countQuery = createQuery(predicate);
JPQLQuery<T> query = querydsl.applyPagination(pageable, createQuery(predicate).select(path));
long total = countQuery.fetchCount();
List<T> content = pageable == null || total > pageable.getOffset() ? query.fetch() : Collections.<T> emptyList();
return PageableExecutionUtils.getPage(query.fetch(), pageable, new TotalSupplier() {
return new PageImpl<T>(content, pageable, total);
@Override
public long get() {
return countQuery.fetchCount();
}
});
}
/*

15
src/main/java/org/springframework/data/jpa/repository/support/SimpleJpaRepository.java
View File

@@ -54,6 +54,8 @@ import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.query.Jpa21Utils;
import org.springframework.data.jpa.repository.query.JpaEntityGraph;
import org.springframework.data.jpa.repository.query.QueryUtils;
import org.springframework.data.repository.support.PageableExecutionUtils;
import org.springframework.data.repository.support.PageableExecutionUtils.TotalSupplier;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.Assert;
@@ -578,16 +580,19 @@ public class SimpleJpaRepository<T, ID extends Serializable>
* @param pageable can be {@literal null}.
* @return
*/
protected <S extends T> Page<S> readPage(TypedQuery<S> query, Class<S> domainClass, Pageable pageable,
Specification<S> spec) {
protected <S extends T> Page<S> readPage(TypedQuery<S> query, final Class<S> domainClass, Pageable pageable,
final Specification<S> spec) {
query.setFirstResult(pageable.getOffset());
query.setMaxResults(pageable.getPageSize());
Long total = executeCountQuery(getCountQuery(spec, domainClass));
List<S> content = total > pageable.getOffset() ? query.getResultList() : Collections.<S> emptyList();
return PageableExecutionUtils.getPage(query.getResultList(), pageable, new TotalSupplier() {
return new PageImpl<S>(content, pageable, total);
@Override
public long get() {
return executeCountQuery(getCountQuery(spec, domainClass));
}
});
}
/**

39
src/test/java/org/springframework/data/jpa/repository/UserRepositoryTests.java
View File

@@ -21,8 +21,8 @@ import static org.junit.Assert.*;
import static org.springframework.data.domain.Example.*;
import static org.springframework.data.domain.ExampleMatcher.*;
import static org.springframework.data.domain.Sort.Direction.*;
import static org.springframework.data.jpa.domain.Specifications.*;
import static org.springframework.data.jpa.domain.Specifications.not;
import static org.springframework.data.jpa.domain.Specifications.*;
import static org.springframework.data.jpa.domain.sample.UserSpecifications.*;
import java.util.ArrayList;
@@ -55,8 +55,7 @@ import org.springframework.dao.IncorrectResultSizeDataAccessException;
import org.springframework.dao.InvalidDataAccessApiUsageException;
import org.springframework.data.domain.Example;
import org.springframework.data.domain.ExampleMatcher;
import org.springframework.data.domain.ExampleMatcher.GenericPropertyMatcher;
import org.springframework.data.domain.ExampleMatcher.StringMatcher;
import org.springframework.data.domain.ExampleMatcher.*;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
@@ -1653,6 +1652,40 @@ public class UserRepositoryTests {
assertThat(secondPage.getContent(), hasItems(youngest3));
}
/**
* @see DATAJPA-912
*/
@Test
public void pageableQueryReportsTotalFromResult() {
flushTestUsers();
Page<User> firstPage = repository.findAll(new PageRequest(0, 10));
assertThat(firstPage.getContent(), hasSize(4));
assertThat(firstPage.getTotalElements(), is(4L));
Page<User> secondPage = repository.findAll(new PageRequest(1, 3));
assertThat(secondPage.getContent(), hasSize(1));
assertThat(secondPage.getTotalElements(), is(4L));
}
/**
* @see DATAJPA-912
*/
@Test
public void pageableQueryReportsTotalFromCount() {
flushTestUsers();
Page<User> firstPage = repository.findAll(new PageRequest(0, 4));
assertThat(firstPage.getContent(), hasSize(4));
assertThat(firstPage.getTotalElements(), is(4L));
Page<User> secondPage = repository.findAll(new PageRequest(10, 10));
assertThat(secondPage.getContent(), hasSize(0));
assertThat(secondPage.getTotalElements(), is(4L));
}
/**
* @see DATAJPA-506
*/

89
src/test/java/org/springframework/data/jpa/repository/query/JpaQueryExecutionUnitTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2008-2014 the original author or authors.
* Copyright 2008-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import static org.mockito.Matchers.*;
import static org.mockito.Mockito.*;
import java.util.Arrays;
import java.util.Collections;
import javax.persistence.EntityManager;
import javax.persistence.Query;
@@ -43,6 +44,7 @@ import org.springframework.data.repository.query.Parameters;
*
* @author Oliver Gierke
* @author Thomas Darimont
* @author Mark Paluch
*/
@RunWith(MockitoJUnitRunner.class)
public class JpaQueryExecutionUnitTests {
@@ -115,9 +117,10 @@ public class JpaQueryExecutionUnitTests {
/**
* @see DATAJPA-124
* @see DATAJPA-912
*/
@Test
public void pagedExecutionDoesNotRetrieveObjectsForPageableOutOfRange() throws Exception {
public void pagedExecutionRetrievesObjectsForPageableOutOfRange() throws Exception {
Parameters<?, ?> parameters = new DefaultParameters(getClass().getMethod("sampleMethod", Pageable.class));
when(jpaQuery.createCountQuery(Mockito.any(Object[].class))).thenReturn(countQuery);
@@ -127,24 +130,94 @@ public class JpaQueryExecutionUnitTests {
PagedExecution execution = new PagedExecution(parameters);
execution.doExecute(jpaQuery, new Object[] { new PageRequest(2, 10) });
verify(query, times(0)).getResultList();
verify(query).getResultList();
verify(countQuery).getResultList();
}
/**
* @see DATAJPA-477
* @see DATAJPA-912
*/
@Test
public void pagedExecutionShouldNotGenerateUnecessaryQueryIfCountReportedNoResults() throws Exception {
public void pagedExecutionShouldNotGenerateCountQueryIfQueryReportedNoResults() throws Exception {
Parameters<?, ?> parameters = new DefaultParameters(getClass().getMethod("sampleMethod", Pageable.class));
when(jpaQuery.createCountQuery(Mockito.any(Object[].class))).thenReturn(countQuery);
when(countQuery.getResultList()).thenReturn(Arrays.asList(0L));
when(jpaQuery.createQuery(Mockito.any(Object[].class))).thenReturn(query);
when(query.getResultList()).thenReturn(Arrays.asList(0L));
PagedExecution execution = new PagedExecution(parameters);
execution.doExecute(jpaQuery, new Object[] { new PageRequest(0, 10) });
verify(query, times(0)).getResultList();
verify(jpaQuery, times(0)).createQuery((Object[]) any());
verify(countQuery, times(0)).getResultList();
verify(jpaQuery, times(0)).createCountQuery((Object[]) any());
}
/**
* @see DATAJPA-912
*/
@Test
public void pagedExecutionShouldUseCountFromResultIfOffsetIsZeroAndResultsWithinPageSize() throws Exception {
Parameters<?, ?> parameters = new DefaultParameters(getClass().getMethod("sampleMethod", Pageable.class));
when(jpaQuery.createQuery(Mockito.any(Object[].class))).thenReturn(query);
when(query.getResultList()).thenReturn(Arrays.asList(new Object(), new Object(), new Object(), new Object()));
PagedExecution execution = new PagedExecution(parameters);
execution.doExecute(jpaQuery, new Object[] { new PageRequest(0, 10) });
verify(jpaQuery, times(0)).createCountQuery((Object[]) any());
}
/**
* @see DATAJPA-912
*/
@Test
public void pagedExecutionShouldUseCountFromResultWithOffsetAndResultsWithinPageSize() throws Exception {
Parameters<?, ?> parameters = new DefaultParameters(getClass().getMethod("sampleMethod", Pageable.class));
when(jpaQuery.createQuery(Mockito.any(Object[].class))).thenReturn(query);
when(query.getResultList()).thenReturn(Arrays.asList(new Object(), new Object(), new Object(), new Object()));
PagedExecution execution = new PagedExecution(parameters);
execution.doExecute(jpaQuery, new Object[] { new PageRequest(5, 10) });
verify(jpaQuery, times(0)).createCountQuery((Object[]) any());
}
/**
* @see DATAJPA-912
*/
@Test
public void pagedExecutionShouldUseRequestCountFromResultWithOffsetAndResultsHitLowerPageSizeBounds() throws Exception {
Parameters<?, ?> parameters = new DefaultParameters(getClass().getMethod("sampleMethod", Pageable.class));
when(jpaQuery.createQuery(Mockito.any(Object[].class))).thenReturn(query);
when(query.getResultList()).thenReturn(Collections.emptyList());
when(jpaQuery.createCountQuery(Mockito.any(Object[].class))).thenReturn(query);
when(countQuery.getResultList()).thenReturn(Arrays.asList(20L));
PagedExecution execution = new PagedExecution(parameters);
execution.doExecute(jpaQuery, new Object[] { new PageRequest(4, 4) });
verify(jpaQuery).createCountQuery((Object[]) any());
}
/**
* @see DATAJPA-912
*/
@Test
public void pagedExecutionShouldUseRequestCountFromResultWithOffsetAndResultsHitUpperPageSizeBounds() throws Exception {
Parameters<?, ?> parameters = new DefaultParameters(getClass().getMethod("sampleMethod", Pageable.class));
when(jpaQuery.createQuery(Mockito.any(Object[].class))).thenReturn(query);
when(query.getResultList()).thenReturn(Arrays.asList(new Object(), new Object(), new Object(), new Object()));
when(jpaQuery.createCountQuery(Mockito.any(Object[].class))).thenReturn(query);
when(countQuery.getResultList()).thenReturn(Arrays.asList(20L));
PagedExecution execution = new PagedExecution(parameters);
execution.doExecute(jpaQuery, new Object[] { new PageRequest(4, 4) });
verify(jpaQuery).createCountQuery((Object[]) any());
}
public static void sampleMethod(Pageable pageable) {

33
src/test/java/org/springframework/data/jpa/repository/support/QueryDslJpaRepositoryTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2008-2015 the original author or authors.
* Copyright 2008-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -53,6 +53,7 @@ import com.querydsl.core.types.dsl.PathBuilderFactory;
*
* @author Oliver Gierke
* @author Thomas Darimont
* @author Mark Paluch
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration({ "classpath:infrastructure.xml" })
@@ -359,4 +360,34 @@ public class QueryDslJpaRepositoryTests {
public void worksWithNullPageable() {
assertThat(repository.findAll(user.dateOfBirth.isNull(), (Pageable) null).getContent(), hasSize(3));
}
/**
* @see DATAJPA-912
*/
@Test
public void pageableQueryReportsTotalFromResult() {
Page<User> firstPage = repository.findAll(user.dateOfBirth.isNull(), new PageRequest(0, 10));
assertThat(firstPage.getContent(), hasSize(3));
assertThat(firstPage.getTotalElements(), is(3L));
Page<User> secondPage = repository.findAll(user.dateOfBirth.isNull(), new PageRequest(1, 2));
assertThat(secondPage.getContent(), hasSize(1));
assertThat(secondPage.getTotalElements(), is(3L));
}
/**
* @see DATAJPA-912
*/
@Test
public void pageableQueryReportsTotalFromCount() {
Page<User> firstPage = repository.findAll(user.dateOfBirth.isNull(), new PageRequest(0, 3));
assertThat(firstPage.getContent(), hasSize(3));
assertThat(firstPage.getTotalElements(), is(3L));
Page<User> secondPage = repository.findAll(user.dateOfBirth.isNull(), new PageRequest(10, 10));
assertThat(secondPage.getContent(), hasSize(0));
assertThat(secondPage.getTotalElements(), is(3L));
}
}

45
src/test/java/org/springframework/data/jpa/repository/support/SimpleJpaRepositoryUnitTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2011-2015 the original author or authors.
* Copyright 2011-2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,12 +15,11 @@
*/
package org.springframework.data.jpa.repository.support;
import static java.util.Collections.singletonMap;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static java.util.Collections.*;
import static org.mockito.Mockito.*;
import java.io.Serializable;
import java.util.Arrays;
import javax.persistence.EntityGraph;
import javax.persistence.EntityManager;
@@ -44,6 +43,7 @@ import org.springframework.data.repository.CrudRepository;
*
* @author Oliver Gierke
* @author Thomas Darimont
* @author Mark Paluch
*/
@RunWith(MockitoJUnitRunner.class)
public class SimpleJpaRepositoryUnitTests {
@@ -81,14 +81,41 @@ public class SimpleJpaRepositoryUnitTests {
/**
* @see DATAJPA-124
* @see DATAJPA-912
*/
@Test
public void doesNotActuallyRetrieveObjectsForPageableOutOfRange() {
public void retrieveObjectsForPageableOutOfRange() {
when(countQuery.getSingleResult()).thenReturn(20L);
repo.findAll(new PageRequest(2, 10));
verify(query, times(0)).getResultList();
verify(query).getResultList();
}
/**
* @see DATAJPA-912
*/
@Test
public void doesNotRetrieveCountWithoutOffsetAndResultsWithinPageSize() {
when(query.getResultList()).thenReturn(Arrays.asList(new User(), new User()));
repo.findAll(new PageRequest(0, 10));
verify(countQuery, never()).getSingleResult();
}
/**
* @see DATAJPA-912
*/
@Test
public void doesNotRetrieveCountWithOffsetAndResultsWithinPageSize() {
when(query.getResultList()).thenReturn(Arrays.asList(new User(), new User()));
repo.findAll(new PageRequest(2, 10));
verify(countQuery, never()).getSingleResult();
}
/**
@@ -106,7 +133,7 @@ public class SimpleJpaRepositoryUnitTests {
*/
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void shouldPropagateConfiguredEntityGraphToFindOne() throws Exception{
public void shouldPropagateConfiguredEntityGraphToFindOne() throws Exception {
String entityGraphName = "User.detail";
when(entityGraphAnnotation.value()).thenReturn(entityGraphName);
@@ -115,7 +142,7 @@ public class SimpleJpaRepositoryUnitTests {
when(em.getEntityGraph(entityGraphName)).thenReturn((EntityGraph) entityGraph);
when(information.getEntityName()).thenReturn("User");
when(metadata.getMethod()).thenReturn(CrudRepository.class.getMethod("findOne", Serializable.class));
Integer id = 0;
repo.findOne(id);