diff --git a/spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingQueryIntegrationTests.java b/spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingQueryIntegrationTests.java new file mode 100644 index 000000000..758b944d6 --- /dev/null +++ b/spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingQueryIntegrationTests.java @@ -0,0 +1,140 @@ +/* + * Copyright 2006-2007 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.item.database; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.batch.item.database.support.SqlPagingQueryProviderFactoryBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.simple.SimpleJdbcTemplate; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.jdbc.SimpleJdbcTestUtils; + +/** + * @author Dave Syer + * @since 2.1 + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = "JdbcPagingItemReaderCommonTests-context.xml") +public class JdbcPagingQueryIntegrationTests { + + private static Log logger = LogFactory.getLog(JdbcPagingQueryIntegrationTests.class); + + @Autowired + private DataSource dataSource; + + private int maxId; + + private SimpleJdbcTemplate jdbcTemplate; + + private int itemCount = 9; + + private int pageSize = 2; + + @Before + public void init() { + jdbcTemplate = new SimpleJdbcTemplate(dataSource); + maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS"); + for (int i = maxId + 1; i <= itemCount; i++) { + jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i); + } + assertEquals(itemCount, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS")); + } + + @After + public void destroy() { + jdbcTemplate.update("DELETE from T_FOOS where ID>?", maxId); + } + + @Test + public void testQueryFromStart() throws Exception { + + PagingQueryProvider queryProvider = getPagingQueryProvider(); + + int total = SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"); + assertTrue(total > pageSize); + int pages = total / pageSize; + + int count = 0; + + List> list = jdbcTemplate.queryForList(queryProvider.generateFirstPageQuery(pageSize)); + logger.debug("First page result: " + list); + assertEquals(pageSize, list.size()); + count += pageSize; + Object oldValue = -1L; + + while (count < pages * pageSize) { + Object startAfterValue = list.get(pageSize - 1).get(queryProvider.getSortKey()); + assertNotSame(oldValue, startAfterValue); + list = jdbcTemplate.queryForList(queryProvider.generateRemainingPagesQuery(pageSize), startAfterValue); + assertEquals(pageSize, list.size()); + count += pageSize; + oldValue = startAfterValue; + } + + if (count < total) { + Object startAfterValue = list.get(pageSize - 1).get(queryProvider.getSortKey()); + list = jdbcTemplate.queryForList(queryProvider.generateRemainingPagesQuery(pageSize), startAfterValue); + assertEquals(total - pages * pageSize, list.size()); + count += list.size(); + } + + assertEquals(total, count); + + } + + @Test + public void testJumpToItem() throws Exception { + + PagingQueryProvider queryProvider = getPagingQueryProvider(); + + List> list = jdbcTemplate.queryForList(queryProvider.generateJumpToItemQuery(pageSize, + pageSize)); + logger.debug("Jump to page result: " + list); + assertEquals(1, list.size()); + Object startAfterValue = list.get(0).entrySet().iterator().next().getValue(); + list = jdbcTemplate.queryForList(queryProvider.generateRemainingPagesQuery(pageSize), startAfterValue); + assertEquals(pageSize, list.size()); + + } + + protected PagingQueryProvider getPagingQueryProvider() throws Exception { + + SqlPagingQueryProviderFactoryBean factory = new SqlPagingQueryProviderFactoryBean(); + factory.setDataSource(dataSource); + factory.setSelectClause("select ID, NAME, VALUE"); + factory.setFromClause("from T_FOOS"); + factory.setSortKey("ID"); + return (PagingQueryProvider) factory.getObject(); + + } + +} diff --git a/spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingRestartIntegrationTests.java b/spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingRestartIntegrationTests.java index 9ebc966a9..2e76ca95c 100644 --- a/spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingRestartIntegrationTests.java +++ b/spring-batch-infrastructure-tests/src/test/java/org/springframework/batch/item/database/JdbcPagingRestartIntegrationTests.java @@ -53,7 +53,7 @@ import org.springframework.test.jdbc.SimpleJdbcTestUtils; @ContextConfiguration(locations = "JdbcPagingItemReaderCommonTests-context.xml") public class JdbcPagingRestartIntegrationTests { - private static Log logger = LogFactory.getLog(JdbcPagingItemReaderAsyncTests.class); + private static Log logger = LogFactory.getLog(JdbcPagingRestartIntegrationTests.class); @Autowired private DataSource dataSource; @@ -160,7 +160,7 @@ public class JdbcPagingRestartIntegrationTests { }); reader.setPageSize(pageSize); reader.afterPropertiesSet(); - reader.setSaveState(false); + reader.setSaveState(true); return reader; } diff --git a/spring-batch-infrastructure-tests/src/test/resources/batch-derby.properties b/spring-batch-infrastructure-tests/src/test/resources/batch-derby.properties index 9b9b6c74f..f35dc1a11 100644 --- a/spring-batch-infrastructure-tests/src/test/resources/batch-derby.properties +++ b/spring-batch-infrastructure-tests/src/test/resources/batch-derby.properties @@ -2,12 +2,12 @@ # for Derby: batch.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver batch.jdbc.url=jdbc:derby:derby-home/test;create=true -batch.jdbc.user=sa +batch.jdbc.user=app batch.jdbc.password= batch.jdbc.testWhileIdle=false batch.jdbc.validationQuery= batch.drop.script=classpath:/org/springframework/batch/core/schema-drop-derby.sql -batch.schema.script=classpath:/org/springframework/batch/core/schema-derby.sql +batch.schema.script=classpath:org/springframework/batch/item/database/init-foo-schema-derby.sql batch.business.schema.script=classpath:/org/springframework/batch/jms/init.sql batch.data.source.init=true batch.database.incrementer.class=org.springframework.jdbc.support.incrementer.DerbyMaxValueIncrementer diff --git a/spring-batch-infrastructure-tests/src/test/resources/log4j.properties b/spring-batch-infrastructure-tests/src/test/resources/log4j.properties index 8137e5c91..169df5292 100644 --- a/spring-batch-infrastructure-tests/src/test/resources/log4j.properties +++ b/spring-batch-infrastructure-tests/src/test/resources/log4j.properties @@ -6,7 +6,7 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %t %c{1}:%L - %m log4j.category.org.apache.activemq=ERROR # log4j.category.org.springframework=DEBUG -# log4j.category.org.springframework.jdbc=DEBUG +log4j.category.org.springframework.jdbc=DEBUG # log4j.category.org.springframework.jms=DEBUG # log4j.category.org.springframework.batch=DEBUG log4j.category.org.springframework.batch.support=INFO diff --git a/spring-batch-infrastructure-tests/src/test/resources/org/springframework/batch/item/database/init-foo-schema-derby.sql b/spring-batch-infrastructure-tests/src/test/resources/org/springframework/batch/item/database/init-foo-schema-derby.sql new file mode 100644 index 000000000..4e52c399e --- /dev/null +++ b/spring-batch-infrastructure-tests/src/test/resources/org/springframework/batch/item/database/init-foo-schema-derby.sql @@ -0,0 +1,24 @@ +DROP TABLE T_FOOS; +DROP TABLE T_WRITE_FOOS; + +CREATE TABLE T_FOOS ( + ID BIGINT NOT NULL, + NAME VARCHAR(45), + VALUE INTEGER +); + +ALTER TABLE T_FOOS ADD PRIMARY KEY (ID); + +INSERT INTO t_foos (id, name, value) VALUES (1, 'bar1', 1); +INSERT INTO t_foos (id, name, value) VALUES (2, 'bar2', 2); +INSERT INTO t_foos (id, name, value) VALUES (3, 'bar3', 3); +INSERT INTO t_foos (id, name, value) VALUES (4, 'bar4', 4); +INSERT INTO t_foos (id, name, value) VALUES (5, 'bar5', 5); + +CREATE TABLE T_WRITE_FOOS ( + ID BIGINT NOT NULL, + NAME VARCHAR(45), + VALUE INTEGER +); + +ALTER TABLE T_WRITE_FOOS ADD PRIMARY KEY (ID); diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/OraclePagingQueryProvider.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/OraclePagingQueryProvider.java index e8eb4f86f..fe975e980 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/OraclePagingQueryProvider.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/OraclePagingQueryProvider.java @@ -43,7 +43,7 @@ public class OraclePagingQueryProvider extends AbstractSqlPagingQueryProvider { int offset = (page * pageSize); offset = offset==0 ? 1 : offset; return SqlPagingQueryUtils.generateRowNumSqlQuery(this, this.getSortKey() + " AS SORT_KEY", false, - "ROWNUM = " + offset); + "TMP_ROW_NUM = " + offset); } private String buildRowNumClause(int pageSize) { diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/SqlPagingQueryUtils.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/SqlPagingQueryUtils.java index 763424d27..6894801b0 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/SqlPagingQueryUtils.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/database/support/SqlPagingQueryUtils.java @@ -101,7 +101,7 @@ public class SqlPagingQueryUtils { public static String generateRowNumSqlQuery(AbstractSqlPagingQueryProvider provider, String selectClause, boolean remainingPageQuery, String rowNumClause) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT * FROM (SELECT ").append(selectClause); + sql.append("SELECT * FROM (SELECT ").append(selectClause).append(", ROWNUM as TMP_ROW_NUM"); sql.append(" FROM ").append(provider.getFromClause()); buildWhereClause(provider, remainingPageQuery, sql); sql.append(" ORDER BY ").append(provider.getSortKey()); diff --git a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/support/AbstractItemCountingItemStreamItemReader.java b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/support/AbstractItemCountingItemStreamItemReader.java index 1ee240581..a8fec29d2 100644 --- a/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/support/AbstractItemCountingItemStreamItemReader.java +++ b/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/support/AbstractItemCountingItemStreamItemReader.java @@ -136,6 +136,9 @@ public abstract class AbstractItemCountingItemStreamItemReader implements Ite catch (Exception e) { throw new ItemStreamException("Failed to initialize the reader", e); } + if (!isSaveState()) { + return; + } if (executionContext.containsKey(ecSupport.getKey(READ_COUNT_MAX))) { maxItemCount = executionContext.getInt(ecSupport.getKey(READ_COUNT_MAX)); diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/support/OraclePagingQueryProviderTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/support/OraclePagingQueryProviderTests.java index 6983d2504..3e0a7749b 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/support/OraclePagingQueryProviderTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/support/OraclePagingQueryProviderTests.java @@ -1,7 +1,7 @@ package org.springframework.batch.item.database.support; -import org.junit.Test; import org.junit.Assert; +import org.junit.Test; /** * @author Thomas Risberg @@ -15,32 +15,32 @@ public class OraclePagingQueryProviderTests extends AbstractSqlPagingQueryProvid @Test @Override public void testGenerateFirstPageQuery() { - String sql = "SELECT * FROM (SELECT id, name, age FROM foo WHERE bar = 1 ORDER BY id ASC) WHERE ROWNUM <= 100"; + String sql = "SELECT * FROM (SELECT id, name, age, ROWNUM as TMP_ROW_NUM FROM foo WHERE bar = 1 ORDER BY id ASC) WHERE ROWNUM <= 100"; String s = pagingQueryProvider.generateFirstPageQuery(pageSize); Assert.assertEquals("", sql, s); pagingQueryProvider.setWhereClause(""); - String sql2 = "SELECT * FROM (SELECT id, name, age FROM foo ORDER BY id ASC) WHERE ROWNUM <= 100"; + String sql2 = "SELECT * FROM (SELECT id, name, age, ROWNUM as TMP_ROW_NUM FROM foo ORDER BY id ASC) WHERE ROWNUM <= 100"; String s2 = pagingQueryProvider.generateFirstPageQuery(pageSize); Assert.assertEquals("", sql2, s2); } @Test @Override public void testGenerateRemainingPagesQuery() { - String sql = "SELECT * FROM (SELECT id, name, age FROM foo WHERE bar = 1 AND id > ? ORDER BY id ASC) WHERE ROWNUM <= 100"; + String sql = "SELECT * FROM (SELECT id, name, age, ROWNUM as TMP_ROW_NUM FROM foo WHERE bar = 1 AND id > ? ORDER BY id ASC) WHERE ROWNUM <= 100"; String s = pagingQueryProvider.generateRemainingPagesQuery(pageSize); Assert.assertEquals("", sql, s); } @Test @Override public void testGenerateJumpToItemQuery() { - String sql = "SELECT * FROM (SELECT id AS SORT_KEY FROM foo WHERE bar = 1 ORDER BY id ASC) WHERE ROWNUM = 100"; + String sql = "SELECT * FROM (SELECT id AS SORT_KEY, ROWNUM as TMP_ROW_NUM FROM foo WHERE bar = 1 ORDER BY id ASC) WHERE TMP_ROW_NUM = 100"; String s = pagingQueryProvider.generateJumpToItemQuery(145, pageSize); Assert.assertEquals("", sql, s); } @Test @Override public void testGenerateJumpToItemQueryForFirstPage() { - String sql = "SELECT * FROM (SELECT id AS SORT_KEY FROM foo WHERE bar = 1 ORDER BY id ASC) WHERE ROWNUM = 1"; + String sql = "SELECT * FROM (SELECT id AS SORT_KEY, ROWNUM as TMP_ROW_NUM FROM foo WHERE bar = 1 ORDER BY id ASC) WHERE TMP_ROW_NUM = 1"; String s = pagingQueryProvider.generateJumpToItemQuery(45, pageSize); Assert.assertEquals("", sql, s); } diff --git a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/support/SqlPagingQueryUtilsTests.java b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/support/SqlPagingQueryUtilsTests.java index 0f5faf8e4..dcedde22d 100644 --- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/support/SqlPagingQueryUtilsTests.java +++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/support/SqlPagingQueryUtilsTests.java @@ -60,14 +60,14 @@ public class SqlPagingQueryUtilsTests { @Test public void testGenerateRowNumSqlQuery() { AbstractSqlPagingQueryProvider qp = new TestSqlPagingQueryProvider("FOO", "BAR", "ID"); - assertEquals("SELECT * FROM (SELECT FOO FROM BAR ORDER BY ID ASC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils + assertEquals("SELECT * FROM (SELECT FOO, ROWNUM as TMP_ROW_NUM FROM BAR ORDER BY ID ASC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils .generateRowNumSqlQuery(qp, false, "ROWNUMBER <= 100")); - assertEquals("SELECT * FROM (SELECT FOO FROM BAR WHERE ID > ? ORDER BY ID ASC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils + assertEquals("SELECT * FROM (SELECT FOO, ROWNUM as TMP_ROW_NUM FROM BAR WHERE ID > ? ORDER BY ID ASC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils .generateRowNumSqlQuery(qp, true, "ROWNUMBER <= 100")); qp.setWhereClause("BAZ IS NOT NULL"); - assertEquals("SELECT * FROM (SELECT FOO FROM BAR WHERE BAZ IS NOT NULL ORDER BY ID ASC) WHERE ROWNUMBER <= 100", + assertEquals("SELECT * FROM (SELECT FOO, ROWNUM as TMP_ROW_NUM FROM BAR WHERE BAZ IS NOT NULL ORDER BY ID ASC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils.generateRowNumSqlQuery(qp, false, "ROWNUMBER <= 100")); - assertEquals("SELECT * FROM (SELECT FOO FROM BAR WHERE BAZ IS NOT NULL AND ID > ? ORDER BY ID ASC) WHERE ROWNUMBER <= 100", + assertEquals("SELECT * FROM (SELECT FOO, ROWNUM as TMP_ROW_NUM FROM BAR WHERE BAZ IS NOT NULL AND ID > ? ORDER BY ID ASC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils.generateRowNumSqlQuery(qp, true, "ROWNUMBER <= 100")); } @@ -90,14 +90,14 @@ public class SqlPagingQueryUtilsTests { public void testGenerateRowNumSqlQueryDescending() { AbstractSqlPagingQueryProvider qp = new TestSqlPagingQueryProvider("FOO", "BAR", "ID"); qp.setAscending(false); - assertEquals("SELECT * FROM (SELECT FOO FROM BAR ORDER BY ID DESC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils + assertEquals("SELECT * FROM (SELECT FOO, ROWNUM as TMP_ROW_NUM FROM BAR ORDER BY ID DESC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils .generateRowNumSqlQuery(qp, false, "ROWNUMBER <= 100")); - assertEquals("SELECT * FROM (SELECT FOO FROM BAR WHERE ID < ? ORDER BY ID DESC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils + assertEquals("SELECT * FROM (SELECT FOO, ROWNUM as TMP_ROW_NUM FROM BAR WHERE ID < ? ORDER BY ID DESC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils .generateRowNumSqlQuery(qp, true, "ROWNUMBER <= 100")); qp.setWhereClause("BAZ IS NOT NULL"); - assertEquals("SELECT * FROM (SELECT FOO FROM BAR WHERE BAZ IS NOT NULL ORDER BY ID DESC) WHERE ROWNUMBER <= 100", + assertEquals("SELECT * FROM (SELECT FOO, ROWNUM as TMP_ROW_NUM FROM BAR WHERE BAZ IS NOT NULL ORDER BY ID DESC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils.generateRowNumSqlQuery(qp, false, "ROWNUMBER <= 100")); - assertEquals("SELECT * FROM (SELECT FOO FROM BAR WHERE BAZ IS NOT NULL AND ID < ? ORDER BY ID DESC) WHERE ROWNUMBER <= 100", + assertEquals("SELECT * FROM (SELECT FOO, ROWNUM as TMP_ROW_NUM FROM BAR WHERE BAZ IS NOT NULL AND ID < ? ORDER BY ID DESC) WHERE ROWNUMBER <= 100", SqlPagingQueryUtils.generateRowNumSqlQuery(qp, true, "ROWNUMBER <= 100")); } diff --git a/spring-batch-samples/src/main/resources/batch-derby.properties b/spring-batch-samples/src/main/resources/batch-derby.properties index 380004cd1..ba9091b3e 100644 --- a/spring-batch-samples/src/main/resources/batch-derby.properties +++ b/spring-batch-samples/src/main/resources/batch-derby.properties @@ -2,7 +2,7 @@ # for Derby: batch.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver batch.jdbc.url=jdbc:derby:derby-home/test;create=true -batch.jdbc.user=sa +batch.jdbc.user=app batch.jdbc.password= batch.jdbc.testWhileIdle=false batch.jdbc.validationQuery= @@ -12,5 +12,6 @@ batch.business.schema.script=business-schema-derby.sql batch.data.source.init=true batch.database.incrementer.class=org.springframework.jdbc.support.incrementer.DerbyMaxValueIncrementer batch.database.incrementer.parent=columnIncrementerParent +batch.lob.handler.class=org.springframework.jdbc.support.lob.DefaultLobHandler batch.grid.size=2 batch.verify.cursor.position=false \ No newline at end of file