BATCH-2163: Add support for SQLite.

Adds a DatabaseType, scripts, a DataFieldMaxValueIncrementer and
a PagingQueryProvider for SQLite.
This commit is contained in:
Luke Taylor
2014-01-13 19:03:21 +00:00
committed by Michael Minella
parent f5a31a99a2
commit 2c007deaf6
17 changed files with 444 additions and 14 deletions

View File

@@ -279,6 +279,19 @@
<fileset dir="${basedir}/src/main/sql" includes="schema*.sql.vpp" />
<mapper type="glob" from="*.sql.vpp" to="*-sybase.sql" />
</vppcopy>
<vppcopy todir="${basedir}/target/generated-resources" overwrite="true">
<config>
<context>
<property key="includes" value="src/main/sql" />
<property file="${basedir}/src/main/sql/sqlite.properties" />
</context>
<engine>
<property key="velocimacro.library" value="src/main/sql/sqlite.vpp" />
</engine>
</config>
<fileset dir="${basedir}/src/main/sql" includes="schema*.sql.vpp" />
<mapper type="glob" from="*.sql.vpp" to="*-sqlite.sql" />
</vppcopy>
</tasks>
</configuration>
<goals>

View File

@@ -0,0 +1,12 @@
-- Autogenerated: do not edit this file
DROP TABLE IF EXISTS BATCH_STEP_EXECUTION_CONTEXT ;
DROP TABLE IF EXISTS BATCH_JOB_EXECUTION_CONTEXT ;
DROP TABLE IF EXISTS BATCH_JOB_EXECUTION_PARAMS ;
DROP TABLE IF EXISTS BATCH_STEP_EXECUTION ;
DROP TABLE IF EXISTS BATCH_JOB_EXECUTION ;
DROP TABLE IF EXISTS BATCH_JOB_PARAMS ;
DROP TABLE IF EXISTS BATCH_JOB_INSTANCE ;
DROP TABLE IF EXISTS BATCH_STEP_EXECUTION_SEQ ;
DROP TABLE IF EXISTS BATCH_JOB_EXECUTION_SEQ ;
DROP TABLE IF EXISTS BATCH_JOB_SEQ ;

View File

@@ -0,0 +1,86 @@
-- Autogenerated: do not edit this file
CREATE TABLE BATCH_JOB_INSTANCE (
JOB_INSTANCE_ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
VERSION INTEGER ,
JOB_NAME VARCHAR(100) NOT NULL,
JOB_KEY VARCHAR(32) NOT NULL,
constraint JOB_INST_UN unique (JOB_NAME, JOB_KEY)
) ;
CREATE TABLE BATCH_JOB_EXECUTION (
JOB_EXECUTION_ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
VERSION INTEGER ,
JOB_INSTANCE_ID INTEGER NOT NULL,
CREATE_TIME TIMESTAMP NOT NULL,
START_TIME TIMESTAMP DEFAULT NULL ,
END_TIME TIMESTAMP DEFAULT NULL ,
STATUS VARCHAR(10) ,
EXIT_CODE VARCHAR(100) ,
EXIT_MESSAGE VARCHAR(2500) ,
LAST_UPDATED TIMESTAMP,
constraint JOB_INST_EXEC_FK foreign key (JOB_INSTANCE_ID)
references BATCH_JOB_INSTANCE(JOB_INSTANCE_ID)
) ;
CREATE TABLE BATCH_JOB_EXECUTION_PARAMS (
JOB_EXECUTION_ID INTEGER NOT NULL ,
TYPE_CD VARCHAR(6) NOT NULL ,
KEY_NAME VARCHAR(100) NOT NULL ,
STRING_VAL VARCHAR(250) ,
DATE_VAL TIMESTAMP DEFAULT NULL ,
LONG_VAL INTEGER ,
DOUBLE_VAL DOUBLE PRECISION ,
IDENTIFYING CHAR(1) NOT NULL ,
constraint JOB_EXEC_PARAMS_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
) ;
CREATE TABLE BATCH_STEP_EXECUTION (
STEP_EXECUTION_ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
VERSION INTEGER NOT NULL,
STEP_NAME VARCHAR(100) NOT NULL,
JOB_EXECUTION_ID INTEGER NOT NULL,
START_TIME TIMESTAMP NOT NULL ,
END_TIME TIMESTAMP DEFAULT NULL ,
STATUS VARCHAR(10) ,
COMMIT_COUNT INTEGER ,
READ_COUNT INTEGER ,
FILTER_COUNT INTEGER ,
WRITE_COUNT INTEGER ,
READ_SKIP_COUNT INTEGER ,
WRITE_SKIP_COUNT INTEGER ,
PROCESS_SKIP_COUNT INTEGER ,
ROLLBACK_COUNT INTEGER ,
EXIT_CODE VARCHAR(100) ,
EXIT_MESSAGE VARCHAR(2500) ,
LAST_UPDATED TIMESTAMP,
constraint JOB_EXEC_STEP_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
) ;
CREATE TABLE BATCH_STEP_EXECUTION_CONTEXT (
STEP_EXECUTION_ID INTEGER NOT NULL PRIMARY KEY,
SHORT_CONTEXT VARCHAR(2500) NOT NULL,
SERIALIZED_CONTEXT CLOB ,
constraint STEP_EXEC_CTX_FK foreign key (STEP_EXECUTION_ID)
references BATCH_STEP_EXECUTION(STEP_EXECUTION_ID)
) ;
CREATE TABLE BATCH_JOB_EXECUTION_CONTEXT (
JOB_EXECUTION_ID INTEGER NOT NULL PRIMARY KEY,
SHORT_CONTEXT VARCHAR(2500) NOT NULL,
SERIALIZED_CONTEXT CLOB ,
constraint JOB_EXEC_CTX_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
) ;
CREATE TABLE BATCH_STEP_EXECUTION_SEQ (
ID INTEGER PRIMARY KEY AUTOINCREMENT
);
CREATE TABLE BATCH_JOB_EXECUTION_SEQ (
ID INTEGER PRIMARY KEY AUTOINCREMENT
);
CREATE TABLE BATCH_JOB_SEQ (
ID INTEGER PRIMARY KEY AUTOINCREMENT
);

View File

@@ -1,5 +1,6 @@
DROP TABLE $!{IFEXISTSBEFORE} BATCH_STEP_EXECUTION_CONTEXT $!{IFEXISTS};
DROP TABLE $!{IFEXISTSBEFORE} BATCH_JOB_EXECUTION_CONTEXT $!{IFEXISTS};
DROP TABLE $!{IFEXISTSBEFORE} BATCH_JOB_EXECUTION_PARAMS $!{IFEXISTS};
DROP TABLE $!{IFEXISTSBEFORE} BATCH_STEP_EXECUTION $!{IFEXISTS};
DROP TABLE $!{IFEXISTSBEFORE} BATCH_JOB_EXECUTION $!{IFEXISTS};
DROP TABLE $!{IFEXISTSBEFORE} BATCH_JOB_PARAMS $!{IFEXISTS};

View File

@@ -0,0 +1,14 @@
platform=sqlite
# SQL language oddities
BIGINT = INTEGER
IDENTITY =
GENERATED = AUTOINCREMENT
IFEXISTSBEFORE = IF EXISTS
DOUBLE = DOUBLE PRECISION
BLOB = BLOB
CLOB = CLOB
TIMESTAMP = TIMESTAMP
CHAR = CHAR
VARCHAR = VARCHAR
# for generating drop statements...
SEQUENCE = TABLE

View File

@@ -0,0 +1,5 @@
#macro (sequence $name $value)CREATE TABLE ${name} (
ID INTEGER PRIMARY KEY AUTOINCREMENT
);
#end
#macro (notnull $name $type)ALTER COLUMN ${name} ${type} NOT NULL#end

View File

@@ -95,6 +95,11 @@
<artifactId>h2</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>

View File

@@ -23,6 +23,7 @@ import static org.springframework.batch.support.DatabaseType.HSQL;
import static org.springframework.batch.support.DatabaseType.MYSQL;
import static org.springframework.batch.support.DatabaseType.ORACLE;
import static org.springframework.batch.support.DatabaseType.POSTGRES;
import static org.springframework.batch.support.DatabaseType.SQLITE;
import static org.springframework.batch.support.DatabaseType.SQLSERVER;
import static org.springframework.batch.support.DatabaseType.SYBASE;
@@ -101,6 +102,9 @@ public class DefaultDataFieldMaxValueIncrementerFactory implements DataFieldMaxV
else if (databaseType == POSTGRES) {
return new PostgreSQLSequenceMaxValueIncrementer(dataSource, incrementerName);
}
else if (databaseType == SQLITE) {
return new SqliteMaxValueIncrementer(dataSource, incrementerName, incrementerColumnName);
}
else if (databaseType == SQLSERVER) {
return new SqlServerMaxValueIncrementer(dataSource, incrementerName, incrementerColumnName);
}

View File

@@ -23,6 +23,7 @@ import static org.springframework.batch.support.DatabaseType.HSQL;
import static org.springframework.batch.support.DatabaseType.MYSQL;
import static org.springframework.batch.support.DatabaseType.ORACLE;
import static org.springframework.batch.support.DatabaseType.POSTGRES;
import static org.springframework.batch.support.DatabaseType.SQLITE;
import static org.springframework.batch.support.DatabaseType.SQLSERVER;
import static org.springframework.batch.support.DatabaseType.SYBASE;
@@ -77,6 +78,7 @@ public class SqlPagingQueryProviderFactoryBean implements FactoryBean {
providers.put(MYSQL,new MySqlPagingQueryProvider());
providers.put(ORACLE,new OraclePagingQueryProvider());
providers.put(POSTGRES,new PostgresPagingQueryProvider());
providers.put(SQLITE, new SqlitePagingQueryProvider());
providers.put(SQLSERVER,new SqlServerPagingQueryProvider());
providers.put(SYBASE,new SybasePagingQueryProvider());
}

View File

@@ -0,0 +1,50 @@
package org.springframework.batch.item.database.support;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.jdbc.datasource.DataSourceUtils;
import org.springframework.jdbc.support.JdbcUtils;
import org.springframework.jdbc.support.incrementer.AbstractColumnMaxValueIncrementer;
/**
* Implemented as a package-private class since it is required for SQLite support, but
* should ideally be in Spring JDBC.
*
* @author Luke Taylor
*/
class SqliteMaxValueIncrementer extends AbstractColumnMaxValueIncrementer {
public SqliteMaxValueIncrementer(DataSource dataSource, String incrementerName, String columnName) {
super(dataSource, incrementerName, columnName);
}
@Override
protected long getNextKey() {
Connection con = DataSourceUtils.getConnection(getDataSource());
Statement stmt = null;
try {
stmt = con.createStatement();
DataSourceUtils.applyTransactionTimeout(stmt, getDataSource());
stmt.executeUpdate("insert into " + getIncrementerName() + " values(null)");
ResultSet rs = stmt.executeQuery("select max(rowid) from " + getIncrementerName());
if (!rs.next()) {
throw new DataAccessResourceFailureException("rowid query failed after executing an update");
}
long nextKey = rs.getLong(1);
stmt.executeUpdate("delete from " + getIncrementerName() + " where " + getColumnName() + " < " + nextKey);
return nextKey;
}
catch (SQLException ex) {
throw new DataAccessResourceFailureException("Could not obtain rowid", ex);
}
finally {
JdbcUtils.closeStatement(stmt);
DataSourceUtils.releaseConnection(con, getDataSource());
}
}
}

View File

@@ -0,0 +1,58 @@
/*
* Copyright 2006-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.database.support;
import org.springframework.util.StringUtils;
/**
* SQLite implementation of a {@link org.springframework.batch.item.database.PagingQueryProvider} using database specific
* features.
*
* @author Luke Taylor
* @since 2.2.5
*/
public class SqlitePagingQueryProvider extends AbstractSqlPagingQueryProvider {
@Override
public String generateFirstPageQuery(int pageSize) {
return SqlPagingQueryUtils.generateLimitSqlQuery(this, false, buildLimitClause(pageSize));
}
@Override
public String generateRemainingPagesQuery(int pageSize) {
if(StringUtils.hasText(getGroupClause())) {
return SqlPagingQueryUtils.generateLimitGroupedSqlQuery(this, true, buildLimitClause(pageSize));
}
else {
return SqlPagingQueryUtils.generateLimitSqlQuery(this, true, buildLimitClause(pageSize));
}
}
@Override
public String generateJumpToItemQuery(int itemIndex, int pageSize) {
int page = itemIndex / pageSize;
int offset = (page * pageSize) - 1;
offset = offset<0 ? 0 : offset;
String limitClause = new StringBuilder().append("LIMIT ").append(offset).append(", 1").toString();
return SqlPagingQueryUtils.generateLimitJumpToQuery(this, limitClause);
}
private String buildLimitClause(int pageSize) {
return new StringBuilder().append("LIMIT ").append(pageSize).toString();
}
}

View File

@@ -29,24 +29,26 @@ import org.springframework.jdbc.support.MetaDataAccessException;
* Enum representing a database type, such as DB2 or oracle. The type also
* contains a product name, which is expected to be the same as the product name
* provided by the database driver's metadata.
*
*
* @author Lucas Ward
* @since 2.0
*/
public enum DatabaseType {
DERBY("Apache Derby"),
DB2("DB2"),
DB2ZOS("DB2ZOS"),
DERBY("Apache Derby"),
DB2("DB2"),
DB2ZOS("DB2ZOS"),
HSQL("HSQL Database Engine"),
SQLSERVER("Microsoft SQL Server"),
MYSQL("MySQL"),
ORACLE("Oracle"),
POSTGRES("PostgreSQL"),
SYBASE("Sybase"), H2("H2");
SYBASE("Sybase"),
H2("H2"),
SQLITE("SQLite");
private static final Map<String, DatabaseType> nameMap;
static{
nameMap = new HashMap<String, DatabaseType>();
for(DatabaseType type: values()){
@@ -56,35 +58,35 @@ public enum DatabaseType {
//A description is necessary due to the nature of database descriptions
//in metadata.
private final String productName;
private DatabaseType(String productName) {
this.productName = productName;
}
public String getProductName() {
return productName;
}
/**
* Static method to obtain a DatabaseType from the provided product name.
*
*
* @param productName
* @return DatabaseType for given product name.
* @throws IllegalArgumentException if none is found.
*/
public static DatabaseType fromProductName(String productName){
if(!nameMap.containsKey(productName)){
throw new IllegalArgumentException("DatabaseType not found for product name: [" +
throw new IllegalArgumentException("DatabaseType not found for product name: [" +
productName + "]");
}
else{
return nameMap.get(productName);
}
}
/**
* Convenience method that pulls a database product name from the DataSource's metadata.
*
*
* @param dataSource
* @return DatabaseType
* @throws MetaDataAccessException

View File

@@ -61,6 +61,7 @@ public class DefaultDataFieldMaxValueIncrementerFactoryTests extends TestCase {
assertTrue(factory.isSupportedIncrementerType("hsql"));
assertTrue(factory.isSupportedIncrementerType("sqlserver"));
assertTrue(factory.isSupportedIncrementerType("sybase"));
assertTrue(factory.isSupportedIncrementerType("sqlite"));
}
public void testUnsupportedDatabaseType(){
@@ -124,5 +125,8 @@ public class DefaultDataFieldMaxValueIncrementerFactoryTests extends TestCase {
assertTrue(factory.getIncrementer("sybase", "NAME") instanceof SybaseMaxValueIncrementer);
}
public void testSqlite(){
assertTrue(factory.getIncrementer("sqlite", "NAME") instanceof SqliteMaxValueIncrementer);
}
}

View File

@@ -0,0 +1,49 @@
package org.springframework.batch.item.database.support;
import static org.junit.Assert.assertEquals;
import java.io.File;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.SimpleDriverDataSource;
/**
* @author Luke Taylor
*/
public class SqliteMaxValueIncrementerTests {
static String dbFile;
static SimpleDriverDataSource dataSource;
static JdbcTemplate template;
@BeforeClass
public static void setUp() {
dbFile = System.getProperty("java.io.tmpdir") + File.separator + "batch_sqlite_inc.db";
dataSource = new SimpleDriverDataSource();
dataSource.setDriverClass(org.sqlite.JDBC.class);
dataSource.setUrl("jdbc:sqlite:" + dbFile);
template = new JdbcTemplate(dataSource);
template.execute("create table max_value (id integer primary key autoincrement)");
}
@AfterClass
public static void removeDbFile() {
File db = new File(dbFile);
if (db.exists()) {
db.delete();
}
dataSource = null;
template = null;
}
@Test
public void testNextKey() throws Exception {
SqliteMaxValueIncrementer mvi = new SqliteMaxValueIncrementer(dataSource, "max_value", "id");
assertEquals(1, mvi.getNextKey());
assertEquals(2, mvi.getNextKey());
assertEquals(3, mvi.getNextKey());
assertEquals(1, template.queryForInt("select count(*) from max_value"));
}
}

View File

@@ -0,0 +1,117 @@
/*
* Copyright 2006-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.database.support;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
/**
* @author Thomas Risberg
* @author Michael Minella
* @author Luke Taylor
*/
public class SqlitePagingQueryProviderTests extends AbstractSqlPagingQueryProviderTests {
public SqlitePagingQueryProviderTests() {
pagingQueryProvider = new MySqlPagingQueryProvider();
}
@Test
@Override
public void testGenerateFirstPageQuery() {
String sql = "SELECT id, name, age FROM foo WHERE bar = 1 ORDER BY id ASC LIMIT 100";
String s = pagingQueryProvider.generateFirstPageQuery(pageSize);
assertEquals(sql, s);
}
@Test @Override
public void testGenerateRemainingPagesQuery() {
String sql = "SELECT id, name, age FROM foo WHERE bar = 1 AND ((id > ?)) ORDER BY id ASC LIMIT 100";
String s = pagingQueryProvider.generateRemainingPagesQuery(pageSize);
assertEquals(sql, s);
}
@Test @Override
public void testGenerateJumpToItemQuery() {
String sql = "SELECT id FROM foo WHERE bar = 1 ORDER BY id ASC LIMIT 99, 1";
String s = pagingQueryProvider.generateJumpToItemQuery(145, pageSize);
assertEquals(sql, s);
}
@Test @Override
public void testGenerateJumpToItemQueryForFirstPage() {
String sql = "SELECT id FROM foo WHERE bar = 1 ORDER BY id ASC LIMIT 0, 1";
String s = pagingQueryProvider.generateJumpToItemQuery(45, pageSize);
assertEquals(sql, s);
}
@Override
@Test
public void testGenerateFirstPageQueryWithGroupBy() {
pagingQueryProvider.setGroupClause("dep");
String sql = "SELECT id, name, age FROM foo WHERE bar = 1 GROUP BY dep ORDER BY id ASC LIMIT 100";
String s = pagingQueryProvider.generateFirstPageQuery(pageSize);
assertEquals(sql, s);
}
@Override
@Test
public void testGenerateRemainingPagesQueryWithGroupBy() {
pagingQueryProvider.setGroupClause("dep");
String sql = "SELECT * FROM (SELECT id, name, age FROM foo WHERE bar = 1 GROUP BY dep) AS MAIN_QRY WHERE ((id > ?)) ORDER BY id ASC LIMIT 100";
String s = pagingQueryProvider.generateRemainingPagesQuery(pageSize);
assertEquals(sql, s);
}
@Override
@Test
public void testGenerateJumpToItemQueryWithGroupBy() {
pagingQueryProvider.setGroupClause("dep");
String sql = "SELECT id FROM foo WHERE bar = 1 GROUP BY dep ORDER BY id ASC LIMIT 99, 1";
String s = pagingQueryProvider.generateJumpToItemQuery(145, pageSize);
assertEquals(sql, s);
}
@Override
@Test
public void testGenerateJumpToItemQueryForFirstPageWithGroupBy() {
pagingQueryProvider.setGroupClause("dep");
String sql = "SELECT id FROM foo WHERE bar = 1 GROUP BY dep ORDER BY id ASC LIMIT 0, 1";
String s = pagingQueryProvider.generateJumpToItemQuery(45, pageSize);
assertEquals(sql, s);
}
@Override
public String getFirstPageSqlWithMultipleSortKeys() {
return "SELECT id, name, age FROM foo WHERE bar = 1 ORDER BY name ASC, id DESC LIMIT 100";
}
@Override
public String getRemainingSqlWithMultipleSortKeys() {
return "SELECT id, name, age FROM foo WHERE bar = 1 AND ((name > ?) OR (name = ? AND id < ?)) ORDER BY name ASC, id DESC LIMIT 100";
}
@Override
public String getJumpToItemQueryWithMultipleSortKeys() {
return "SELECT name, id FROM foo WHERE bar = 1 ORDER BY name ASC, id DESC LIMIT 99, 1";
}
@Override
public String getJumpToItemQueryForFirstPageWithMultipleSortKeys() {
return "SELECT name, id FROM foo WHERE bar = 1 ORDER BY name ASC, id DESC LIMIT 0, 1";
}
}

View File

@@ -8,6 +8,7 @@ import static org.springframework.batch.support.DatabaseType.HSQL;
import static org.springframework.batch.support.DatabaseType.MYSQL;
import static org.springframework.batch.support.DatabaseType.ORACLE;
import static org.springframework.batch.support.DatabaseType.POSTGRES;
import static org.springframework.batch.support.DatabaseType.SQLITE;
import static org.springframework.batch.support.DatabaseType.SQLSERVER;
import static org.springframework.batch.support.DatabaseType.SYBASE;
import static org.springframework.batch.support.DatabaseType.fromProductName;
@@ -36,6 +37,7 @@ public class DatabaseTypeTests {
assertEquals(ORACLE, fromProductName("Oracle"));
assertEquals(POSTGRES, fromProductName("PostgreSQL"));
assertEquals(SYBASE, fromProductName("Sybase"));
assertEquals(SQLITE, fromProductName("SQLite"));
}
@Test(expected = IllegalArgumentException.class)

View File

@@ -537,6 +537,12 @@
<version>1.2.125</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.7.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>