Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BATCH-1860: Use sub query for Derby for JdbcPagingItemReader

Browse Source 
Order the results before the ROW_NUM restriction is applied (and make related infrastructure tests more robust)
This commit is contained in:
David Thexton
2012-06-07 16:00:53 +01:00
committed by Dave Syer
parent 1c843e2ded
commit 8c445bcb08
13 changed files with 68 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/IbatisPagingItemReaderAsyncTests.java
View File

@@ -57,7 +57,7 @@ public class IbatisPagingItemReaderAsyncTests {
public void init() {
SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS");
for (int i = maxId + 1; i <= ITEM_COUNT; i++) {
for (int i = ITEM_COUNT; i > maxId; i--) {
jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i);
}
assertEquals(ITEM_COUNT, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"));

4
spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingItemReaderAsyncTests.java
View File

@@ -63,7 +63,7 @@ public class JdbcPagingItemReaderAsyncTests {
public void init() {
SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS");
for (int i = maxId + 1; i <= ITEM_COUNT; i++) {
for (int i = ITEM_COUNT; i > maxId; i--) {
jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i);
}
assertEquals(ITEM_COUNT, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"));
@@ -141,7 +141,7 @@ public class JdbcPagingItemReaderAsyncTests {
factory.setDataSource(dataSource);
factory.setSelectClause("select ID, NAME, VALUE");
factory.setFromClause("from T_FOOS");
factory.setSortKey("ID");
factory.setSortKey("VALUE");
reader.setQueryProvider((PagingQueryProvider) factory.getObject());
reader.setRowMapper(new ParameterizedRowMapper<Foo>() {
public Foo mapRow(ResultSet rs, int i) throws SQLException {

2
spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingQueryIntegrationTests.java
View File

@@ -63,7 +63,7 @@ public class JdbcPagingQueryIntegrationTests {
public void init() {
jdbcTemplate = new SimpleJdbcTemplate(dataSource);
maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS");
for (int i = maxId + 1; i <= itemCount; i++) {
for (int i = itemCount; i > maxId; i--) {
jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i);
}
assertEquals(itemCount, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"));

4
spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingRestartIntegrationTests.java
View File

@@ -70,7 +70,7 @@ public class JdbcPagingRestartIntegrationTests {
public void init() {
jdbcTemplate = new SimpleJdbcTemplate(dataSource);
maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS");
for (int i = maxId + 1; i <= itemCount; i++) {
for (int i = itemCount; i > maxId; i--) {
jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i);
}
assertEquals(itemCount, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"));
@@ -147,7 +147,7 @@ public class JdbcPagingRestartIntegrationTests {
factory.setDataSource(dataSource);
factory.setSelectClause("select ID, NAME, VALUE");
factory.setFromClause("from T_FOOS");
factory.setSortKey("ID");
factory.setSortKey("VALUE");
reader.setQueryProvider((PagingQueryProvider) factory.getObject());
reader.setRowMapper(new ParameterizedRowMapper<Foo>() {
public Foo mapRow(ResultSet rs, int i) throws SQLException {

2
spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JpaPagingItemReaderAsyncTests.java
View File

@@ -60,7 +60,7 @@ public class JpaPagingItemReaderAsyncTests {
public void init() {
SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS");
for (int i = maxId + 1; i <= ITEM_COUNT; i++) {
for (int i = ITEM_COUNT; i > maxId; i--) {
jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i);
}
assertEquals(ITEM_COUNT, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"));

1
spring-batch-infrastructure-tests/src/test/resources/log4j.properties
View File

@@ -12,3 +12,4 @@ log4j.category.org.springframework.jdbc.datasource=INFO
# log4j.category.org.springframework.batch=DEBUG
log4j.category.org.springframework.batch.support=INFO
# log4j.category.org.springframework.retry=DEBUG
# log4j.category.org.springframework.batch.item.database=DEBUG

10
spring-batch-infrastructure-tests/src/test/resources/org/springframework/batch/item/database/init-foo-schema-derby.sql
View File

@@ -9,11 +9,11 @@ CREATE TABLE T_FOOS (
ALTER TABLE T_FOOS ADD PRIMARY KEY (ID);
INSERT INTO t_foos (id, name, value) VALUES (1, 'bar1', 1);
INSERT INTO t_foos (id, name, value) VALUES (2, 'bar2', 2);
INSERT INTO t_foos (id, name, value) VALUES (3, 'bar3', 3);
INSERT INTO t_foos (id, name, value) VALUES (4, 'bar4', 4);
INSERT INTO t_foos (id, name, value) VALUES (5, 'bar5', 5);
INSERT INTO t_foos (id, name, value) VALUES (1, 'bar2', 2);
INSERT INTO t_foos (id, name, value) VALUES (2, 'bar4', 4);
INSERT INTO t_foos (id, name, value) VALUES (3, 'bar1', 1);
INSERT INTO t_foos (id, name, value) VALUES (4, 'bar5', 5);
INSERT INTO t_foos (id, name, value) VALUES (5, 'bar3', 3);
CREATE TABLE T_WRITE_FOOS (
ID BIGINT NOT NULL,

10
spring-batch-infrastructure-tests/src/test/resources/org/springframework/batch/item/database/init-foo-schema-hsqldb.sql
View File

@@ -9,11 +9,11 @@ CREATE TABLE T_FOOS (
ALTER TABLE T_FOOS ADD PRIMARY KEY (ID);
INSERT INTO t_foos (id, name, value) VALUES (1, 'bar1', 1);
INSERT INTO t_foos (id, name, value) VALUES (2, 'bar2', 2);
INSERT INTO t_foos (id, name, value) VALUES (3, 'bar3', 3);
INSERT INTO t_foos (id, name, value) VALUES (4, 'bar4', 4);
INSERT INTO t_foos (id, name, value) VALUES (5, 'bar5', 5);
INSERT INTO t_foos (id, name, value) VALUES (1, 'bar2', 2);
INSERT INTO t_foos (id, name, value) VALUES (2, 'bar4', 4);
INSERT INTO t_foos (id, name, value) VALUES (3, 'bar1', 1);
INSERT INTO t_foos (id, name, value) VALUES (4, 'bar5', 5);
INSERT INTO t_foos (id, name, value) VALUES (5, 'bar3', 3);
CREATE TABLE T_WRITE_FOOS (
ID BIGINT NOT NULL,

10
spring-batch-infrastructure-tests/src/test/resources/org/springframework/batch/item/database/init-foo-schema-postgres.sql
View File

@@ -9,11 +9,11 @@ CREATE TABLE T_FOOS (
ALTER TABLE T_FOOS ADD PRIMARY KEY (ID);
INSERT INTO t_foos (id, name, VALUE) VALUES (1, 'bar1', 1);
INSERT INTO t_foos (id, name, VALUE) VALUES (2, 'bar2', 2);
INSERT INTO t_foos (id, name, VALUE) VALUES (3, 'bar3', 3);
INSERT INTO t_foos (id, name, VALUE) VALUES (4, 'bar4', 4);
INSERT INTO t_foos (id, name, VALUE) VALUES (5, 'bar5', 5);
INSERT INTO t_foos (id, name, value) VALUES (1, 'bar2', 2);
INSERT INTO t_foos (id, name, value) VALUES (2, 'bar4', 4);
INSERT INTO t_foos (id, name, value) VALUES (3, 'bar1', 1);
INSERT INTO t_foos (id, name, value) VALUES (4, 'bar5', 5);
INSERT INTO t_foos (id, name, value) VALUES (5, 'bar3', 3);
CREATE TABLE T_WRITE_FOOS (
ID BIGINT NOT NULL,

26
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/DerbyPagingQueryProvider.java
View File

@@ -5,7 +5,7 @@
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -26,7 +26,11 @@ import javax.sql.DataSource;
* Derby implementation of a {@link PagingQueryProvider} using standard SQL:2003 windowing functions.
* These features are supported starting with Apache Derby version 10.4.1.3.
*
* As the OVER() function does not support the ORDER BY clause a sub query is instead used to order the results
* before the ROW_NUM restriction is applied
*
* @author Thomas Risberg
* @author David Thexton
* @since 2.0
*/
public class DerbyPagingQueryProvider extends SqlWindowingPagingQueryProvider {
@@ -42,23 +46,17 @@ public class DerbyPagingQueryProvider extends SqlWindowingPagingQueryProvider {
}
}
@Override
protected Object getSubQueryAlias() {
return "AS TMP_SUB ";
}
@Override
protected String getOverClause() {
return "";
}
@Override
protected String getAfterWhereClause() {
if (version!=null && "10.6.1".compareTo(version) > 0) {
// Old behaviour retained, even though it is broken
return "";
}
return " " + super.getOverClause();
protected String getOverSubstituteClauseStart() {
return " FROM (SELECT " + getSelectClause();
}
protected String getOverSubstituteClauseEnd() {
return " " + super.getOverClause() + ") AS TMP_ORDERED";
}
}

47
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/SqlWindowingPagingQueryProvider.java
View File

@@ -35,16 +35,21 @@ public class SqlWindowingPagingQueryProvider extends AbstractSqlPagingQueryProvi
sql.append("SELECT ").append(getSelectClause()).append(", ");
sql.append("ROW_NUMBER() OVER (").append(getOverClause());
sql.append(") AS ROW_NUMBER");
sql.append(getOverSubstituteClauseStart());
sql.append(" FROM ").append(getFromClause()).append(
getWhereClause() == null ? "" : " WHERE " + getWhereClause());
sql.append(getAfterWhereClause());
String alias = extractTableAlias();
sql.append(") ").append(getSubQueryAlias()).append("WHERE " + alias + "ROW_NUMBER <= ").append(pageSize);
sql.append(getOverSubstituteClauseEnd());
sql.append(") ").append(getSubQueryAlias()).append("WHERE ").append(extractTableAlias()).append(
"ROW_NUMBER <= ").append(pageSize);
return sql.toString();
}
private String extractTableAlias() {
protected Object getSubQueryAlias() {
return "AS TMP_SUB ";
}
protected Object extractTableAlias() {
String alias = "" + getSubQueryAlias();
if (StringUtils.hasText(alias) && alias.toUpperCase().startsWith("AS")) {
alias = alias.substring(3).trim() + ".";
@@ -52,10 +57,6 @@ public class SqlWindowingPagingQueryProvider extends AbstractSqlPagingQueryProvi
return alias;
}
protected Object getSubQueryAlias() {
return "";
}
@Override
public String generateRemainingPagesQuery(int pageSize) {
StringBuilder sql = new StringBuilder();
@@ -63,6 +64,7 @@ public class SqlWindowingPagingQueryProvider extends AbstractSqlPagingQueryProvi
sql.append("SELECT ").append(getSelectClause()).append(", ");
sql.append("ROW_NUMBER() OVER (").append(getOverClause());
sql.append(") AS ROW_NUMBER");
sql.append(getOverSubstituteClauseStart());
sql.append(" FROM ").append(getFromClause());
sql.append(" WHERE ");
if (getWhereClause() != null) {
@@ -77,17 +79,13 @@ public class SqlWindowingPagingQueryProvider extends AbstractSqlPagingQueryProvi
sql.append(" < ");
}
sql.append(getSortKeyPlaceHolder());
sql.append(getAfterWhereClause());
String alias = extractTableAlias();
sql.append(") ").append(getSubQueryAlias()).append("WHERE " + alias + "ROW_NUMBER <= ").append(pageSize);
sql.append(getOverSubstituteClauseEnd());
sql.append(") ").append(getSubQueryAlias()).append("WHERE ").append(extractTableAlias()).append(
"ROW_NUMBER <= ").append(pageSize);
return sql.toString();
}
protected String getAfterWhereClause() {
return "";
}
@Override
public String generateJumpToItemQuery(int itemIndex, int pageSize) {
int page = itemIndex / pageSize;
@@ -101,11 +99,12 @@ public class SqlWindowingPagingQueryProvider extends AbstractSqlPagingQueryProvi
sql.append("SELECT ").append(getSortKey()).append(" AS SORT_KEY, ");
sql.append("ROW_NUMBER() OVER (").append(getOverClause());
sql.append(") AS ROW_NUMBER");
sql.append(" FROM ").append(getFromClause()).append(
getWhereClause() == null ? "" : " WHERE " + getWhereClause());
sql.append(getAfterWhereClause());
String alias = extractTableAlias();
sql.append(") ").append(getSubQueryAlias()).append("WHERE " + alias + "ROW_NUMBER = ").append(lastRowNum);
sql.append(getOverSubstituteClauseStart());
sql.append(" FROM ").append(getFromClause());
sql.append(getWhereClause() == null ? "" : " WHERE " + getWhereClause());
sql.append(getOverSubstituteClauseEnd());
sql.append(") ").append(getSubQueryAlias()).append("WHERE ").append(extractTableAlias()).append(
"ROW_NUMBER = ").append(lastRowNum);
return sql.toString();
}
@@ -114,6 +113,14 @@ public class SqlWindowingPagingQueryProvider extends AbstractSqlPagingQueryProvi
return "ORDER BY " + getSortKey() + " " + getAscendingClause();
}
protected String getOverSubstituteClauseStart() {
return "";
}
protected String getOverSubstituteClauseEnd() {
return "";
}
private String getAscendingClause() {
if (isAscending()) {
return "ASC";

5
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/SybasePagingQueryProvider.java
View File

@@ -37,6 +37,11 @@ public class SybasePagingQueryProvider extends SqlWindowingPagingQueryProvider {
return SqlPagingQueryUtils.generateTopSqlQuery(this, true, buildTopClause(pageSize));
}
@Override
protected Object getSubQueryAlias() {
return "";
}
private String buildTopClause(int pageSize) {
return new StringBuilder().append("TOP ").append(pageSize).toString();
}

2
spring-batch-parent/pom.xml
View File

@@ -519,7 +519,7 @@
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<version>10.6.2.1</version>
<version>10.8.2.2</version>
<scope>test</scope>
</dependency>
<dependency>