RESOLVED - issue BATCH-1389: Thread safety in JdbcPagingReader

http://jira.springframework.org/browse/BATCH-1389
2009-09-04 13:19:10 +00:00
parent 0cc77cf6ca
commit 29fb16b522
7 changed files with 673 additions and 145 deletions
--- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/IbatisPagingItemReaderAsyncTests.java
+++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/IbatisPagingItemReaderAsyncTests.java
@@ -0,0 +1,153 @@
+package org.springframework.batch.item.database;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.item.sample.Foo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
+import org.springframework.orm.ibatis.SqlMapClientFactoryBean;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.test.jdbc.SimpleJdbcTestUtils;
+
+import com.ibatis.sqlmap.client.SqlMapClient;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = "JdbcPagingItemReaderCommonTests-context.xml")
+public class IbatisPagingItemReaderAsyncTests {
+
+	/**
+	 * The number of items to read
+	 */
+	private static final int ITEM_COUNT = 100;
+
+	/**
+	 * The number of threads to create
+	 */
+	private static final int THREAD_COUNT = 5;
+
+	private static Log logger = LogFactory.getLog(IbatisPagingItemReaderAsyncTests.class);
+
+	@Autowired
+	private DataSource dataSource;
+
+	private int maxId;
+
+	@Before
+	public void init() {
+		SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
+		maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS");
+		for (int i = maxId + 1; i <= ITEM_COUNT; i++) {
+			jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i);
+		}
+		assertEquals(ITEM_COUNT, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"));
+	}
+
+	@After
+	public void destroy() {
+		SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
+		jdbcTemplate.update("DELETE from T_FOOS where ID>?", maxId);
+	}
+
+	@Test
+	public void testAsyncReader() throws Throwable {
+		List<Throwable> throwables = new ArrayList<Throwable>();
+		int max = 10;
+		for (int i = 0; i < max; i++) {
+			try {
+				doTest();
+			}
+			catch (Throwable e) {
+				throwables.add(e);
+			}
+		}
+		if (!throwables.isEmpty()) {
+			throw new IllegalStateException(String.format("Failed %d out of %d", throwables.size(), max), throwables
+					.get(0));
+		}
+	}
+
+	/**
+	 * @throws Exception
+	 * @throws InterruptedException
+	 * @throws ExecutionException
+	 */
+	private void doTest() throws Exception, InterruptedException, ExecutionException {
+		final IbatisPagingItemReader<Foo> reader = getItemReader();
+		CompletionService<List<Foo>> completionService = new ExecutorCompletionService<List<Foo>>(Executors
+				.newFixedThreadPool(THREAD_COUNT));
+		for (int i = 0; i < THREAD_COUNT; i++) {
+			completionService.submit(new Callable<List<Foo>>() {
+				public List<Foo> call() throws Exception {
+					List<Foo> list = new ArrayList<Foo>();
+					Foo next = null;
+					do {
+						next = reader.read();
+						Thread.sleep(10L);
+						logger.debug("Reading item: " + next);
+						if (next != null) {
+							list.add(next);
+						}
+					} while (next != null);
+					return list;
+				}
+			});
+		}
+		int count = 0;
+		for (int i = 0; i < THREAD_COUNT; i++) {
+			List<Foo> items = completionService.take().get();
+			count += items.size();
+			logger.debug("Finished items count: " + items.size());
+			logger.debug("Finished items: " + items);
+			assertNotNull(items);
+		}
+		assertEquals(ITEM_COUNT, count);
+		reader.close();
+	}
+
+	private IbatisPagingItemReader<Foo> getItemReader() throws Exception {
+		SqlMapClientFactoryBean factory = new SqlMapClientFactoryBean();
+		factory.setConfigLocation(new ClassPathResource("ibatis-config.xml", getClass()));
+		factory.setDataSource(dataSource);
+		factory.afterPropertiesSet();
+		SqlMapClient sqlMapClient = createSqlMapClient();
+
+		IbatisPagingItemReader<Foo> reader = new IbatisPagingItemReader<Foo>();
+		reader.setQueryId("getPagedFoos");
+		reader.setPageSize(2);
+		reader.setSqlMapClient(sqlMapClient);
+		reader.setSaveState(true);
+
+		reader.afterPropertiesSet();
+
+		return reader;
+	}
+
+	private SqlMapClient createSqlMapClient() throws Exception {
+		SqlMapClientFactoryBean factory = new SqlMapClientFactoryBean();
+		factory.setConfigLocation(new ClassPathResource("ibatis-config.xml", getClass()));
+		factory.setDataSource(dataSource);
+		factory.afterPropertiesSet();
+		return (SqlMapClient) factory.getObject();
+	}
+
+
+}
--- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/JdbcPagingItemReaderAsyncTests.java
+++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/JdbcPagingItemReaderAsyncTests.java
@@ -0,0 +1,156 @@
+package org.springframework.batch.item.database;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.database.support.HsqlPagingQueryProvider;
+import org.springframework.batch.item.sample.Foo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.simple.ParameterizedRowMapper;
+import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.test.jdbc.SimpleJdbcTestUtils;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = "JdbcPagingItemReaderCommonTests-context.xml")
+public class JdbcPagingItemReaderAsyncTests {
+
+	/**
+	 * The page size
+	 */
+	private static final int PAGE_SIZE = 5;
+
+	/**
+	 * The number of items to read
+	 */
+	private static final int ITEM_COUNT = 100;
+
+	/**
+	 * The number of threads to create
+	 */
+	private static final int THREAD_COUNT = 5;
+
+	private static Log logger = LogFactory.getLog(JdbcPagingItemReaderAsyncTests.class);
+
+	@Autowired
+	private DataSource dataSource;
+
+	private int maxId;
+
+	@Before
+	public void init() {
+		SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
+		maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS");
+		for (int i = maxId + 1; i <= ITEM_COUNT; i++) {
+			jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i);
+		}
+		assertEquals(ITEM_COUNT, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"));
+	}
+
+	@After
+	public void destroy() {
+		SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
+		jdbcTemplate.update("DELETE from T_FOOS where ID>?", maxId);
+	}
+
+	@Test
+	public void testAsyncReader() throws Throwable {
+		List<Throwable> throwables = new ArrayList<Throwable>();
+		int max = 10;
+		for (int i = 0; i < max; i++) {
+			try {
+				doTest();
+			}
+			catch (Throwable e) {
+				throwables.add(e);
+			}
+		}
+		if (!throwables.isEmpty()) {
+			throw new IllegalStateException(String.format("Failed %d out of %d", throwables.size(), max), throwables
+					.get(0));
+		}
+	}
+
+	/**
+	 * @throws Exception
+	 * @throws InterruptedException
+	 * @throws ExecutionException
+	 */
+	private void doTest() throws Exception, InterruptedException, ExecutionException {
+		final ItemReader<Foo> reader = getItemReader();
+		CompletionService<List<Foo>> completionService = new ExecutorCompletionService<List<Foo>>(Executors
+				.newFixedThreadPool(THREAD_COUNT));
+		for (int i = 0; i < THREAD_COUNT; i++) {
+			completionService.submit(new Callable<List<Foo>>() {
+				public List<Foo> call() throws Exception {
+					List<Foo> list = new ArrayList<Foo>();
+					Foo next = null;
+					do {
+						next = reader.read();
+						Thread.sleep(10L);
+						logger.debug("Reading item: " + next);
+						if (next != null) {
+							list.add(next);
+						}
+					} while (next != null);
+					return list;
+				}
+			});
+		}
+		int count = 0;
+		for (int i = 0; i < THREAD_COUNT; i++) {
+			List<Foo> items = completionService.take().get();
+			count += items.size();
+			logger.debug("Finished items count: " + items.size());
+			logger.debug("Finished items: " + items);
+			assertNotNull(items);
+		}
+		assertEquals(ITEM_COUNT, count);
+	}
+
+	protected ItemReader<Foo> getItemReader() throws Exception {
+
+		JdbcPagingItemReader<Foo> reader = new JdbcPagingItemReader<Foo>();
+		reader.setDataSource(dataSource);
+		HsqlPagingQueryProvider queryProvider = new HsqlPagingQueryProvider();
+		queryProvider.setSelectClause("select ID, NAME, VALUE");
+		queryProvider.setFromClause("from T_FOOS");
+		queryProvider.setSortKey("ID");
+		reader.setQueryProvider(queryProvider);
+		reader.setRowMapper(new ParameterizedRowMapper<Foo>() {
+			public Foo mapRow(ResultSet rs, int i) throws SQLException {
+				Foo foo = new Foo();
+				foo.setId(rs.getInt(1));
+				foo.setName(rs.getString(2));
+				foo.setValue(rs.getInt(3));
+				return foo;
+			}
+		});
+		reader.setPageSize(PAGE_SIZE);
+		reader.afterPropertiesSet();
+		reader.setSaveState(false);
+
+		return reader;
+	}
+
+}
--- a/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/JpaPagingItemReaderAsyncTests.java
+++ b/spring-batch-infrastructure/src/test/java/org/springframework/batch/item/database/JpaPagingItemReaderAsyncTests.java
@@ -0,0 +1,142 @@
+package org.springframework.batch.item.database;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+
+import javax.persistence.EntityManagerFactory;
+import javax.sql.DataSource;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.item.ExecutionContext;
+import org.springframework.batch.item.sample.Foo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.test.jdbc.SimpleJdbcTestUtils;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = "JpaPagingItemReaderCommonTests-context.xml")
+public class JpaPagingItemReaderAsyncTests {
+
+	/**
+	 * The number of items to read
+	 */
+	private static final int ITEM_COUNT = 100;
+
+	/**
+	 * The number of threads to create
+	 */
+	private static final int THREAD_COUNT = 5;
+
+	private static Log logger = LogFactory.getLog(JpaPagingItemReaderAsyncTests.class);
+
+	@Autowired
+	private DataSource dataSource;
+
+	@Autowired
+	private EntityManagerFactory entityManagerFactory;
+
+	private int maxId;
+
+	@Before
+	public void init() {
+		SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
+		maxId = jdbcTemplate.queryForInt("SELECT MAX(ID) from T_FOOS");
+		for (int i = maxId + 1; i <= ITEM_COUNT; i++) {
+			jdbcTemplate.update("INSERT into T_FOOS (ID,NAME,VALUE) values (?, ?, ?)", i, "foo" + i, i);
+		}
+		assertEquals(ITEM_COUNT, SimpleJdbcTestUtils.countRowsInTable(jdbcTemplate, "T_FOOS"));
+	}
+
+	@After
+	public void destroy() {
+		SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(dataSource);
+		jdbcTemplate.update("DELETE from T_FOOS where ID>?", maxId);
+	}
+
+	@Test
+	public void testAsyncReader() throws Throwable {
+		List<Throwable> throwables = new ArrayList<Throwable>();
+		int max = 10;
+		for (int i = 0; i < max; i++) {
+			try {
+				doTest();
+			}
+			catch (Throwable e) {
+				throwables.add(e);
+			}
+		}
+		if (!throwables.isEmpty()) {
+			throw new IllegalStateException(String.format("Failed %d out of %d", throwables.size(), max), throwables
+					.get(0));
+		}
+	}
+
+	/**
+	 * @throws Exception
+	 * @throws InterruptedException
+	 * @throws ExecutionException
+	 */
+	private void doTest() throws Exception, InterruptedException, ExecutionException {
+		final JpaPagingItemReader<Foo> reader = getItemReader();
+		CompletionService<List<Foo>> completionService = new ExecutorCompletionService<List<Foo>>(Executors
+				.newFixedThreadPool(THREAD_COUNT));
+		for (int i = 0; i < THREAD_COUNT; i++) {
+			completionService.submit(new Callable<List<Foo>>() {
+				public List<Foo> call() throws Exception {
+					List<Foo> list = new ArrayList<Foo>();
+					Foo next = null;
+					do {
+						next = reader.read();
+						Thread.sleep(10L);
+						logger.debug("Reading item: " + next);
+						if (next != null) {
+							list.add(next);
+						}
+					} while (next != null);
+					return list;
+				}
+			});
+		}
+		int count = 0;
+		for (int i = 0; i < THREAD_COUNT; i++) {
+			List<Foo> items = completionService.take().get();
+			count += items.size();
+			logger.debug("Finished items count: " + items.size());
+			logger.debug("Finished items: " + items);
+			assertNotNull(items);
+		}
+		assertEquals(ITEM_COUNT, count);
+		reader.close();
+	}
+
+	private JpaPagingItemReader<Foo> getItemReader() throws Exception {
+
+		String jpqlQuery = "select f from Foo f";
+
+		JpaPagingItemReader<Foo> reader = new JpaPagingItemReader<Foo>();
+		reader.setQueryString(jpqlQuery);
+		reader.setEntityManagerFactory(entityManagerFactory);
+		reader.setPageSize(3);
+		reader.afterPropertiesSet();
+		reader.setSaveState(false);
+		reader.open(new ExecutionContext());
+
+		return reader;
+	}
+
+}