Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Integrate SAP Hana as supported Spring Batch database

Browse Source 
This commit adds SAP HANA as a supported Spring Batch database, enabling
developers to seamlessly move their existing Spring Batch projects to SAP HANA
or easily starting new Spring Batch projects on SAP HANA.

This commit contains the following changes:

- Add SAP HANA to the DatabaseType enum
- Add HanaPagingQueryProvider and tests
- Add properties files for SAP HANA

Issue #2515
This commit is contained in:
Jonathan Bregler
2018-04-12 13:40:20 +02:00
committed by Mahmoud Ben Hassine
parent 44dd4f7f56
commit 9388abb589
23 changed files with 761 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
spring-batch-core/src/main/resources/batch-hana.properties Normal file
View File

@@ -0,0 +1,17 @@
# Placeholders batch.*
# for SAP HANA:
batch.jdbc.driver=com.sap.db.jdbc.Driver
batch.jdbc.url=jdbc:sap://localhost:39015/
batch.jdbc.user=SPRING_TEST
batch.jdbc.password=Spr1ng_test
batch.database.incrementer.class=org.springframework.jdbc.support.incrementer.HanaSequenceMaxValueIncrementer
batch.schema.script=classpath:/org/springframework/batch/core/schema-hana.sql
batch.drop.script=classpath:/org/springframework/batch/core/schema-drop-hana.sql
batch.jdbc.testWhileIdle=true
batch.jdbc.validationQuery=
# Non-platform dependent settings that you might like to change
batch.data.source.init=true
batch.table.prefix=BATCH_

11
spring-batch-core/src/main/resources/org/springframework/batch/core/schema-drop-hana.sql Normal file
View File

@@ -0,0 +1,11 @@
-- Autogenerated: do not edit this file
DROP TABLE BATCH_STEP_EXECUTION_CONTEXT ;
DROP TABLE BATCH_JOB_EXECUTION_CONTEXT ;
DROP TABLE BATCH_JOB_EXECUTION_PARAMS ;
DROP TABLE BATCH_STEP_EXECUTION ;
DROP TABLE BATCH_JOB_EXECUTION ;
DROP TABLE BATCH_JOB_INSTANCE ;
DROP SEQUENCE BATCH_STEP_EXECUTION_SEQ ;
DROP SEQUENCE BATCH_JOB_EXECUTION_SEQ ;
DROP SEQUENCE BATCH_JOB_SEQ ;

81
spring-batch-core/src/main/resources/org/springframework/batch/core/schema-hana.sql Normal file
View File

@@ -0,0 +1,81 @@
-- Autogenerated: do not edit this file
CREATE TABLE BATCH_JOB_INSTANCE (
JOB_INSTANCE_ID BIGINT NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
VERSION BIGINT ,
JOB_NAME VARCHAR(100) NOT NULL,
JOB_KEY VARCHAR(32) NOT NULL,
constraint JOB_INST_UN unique (JOB_NAME, JOB_KEY)
) ;
CREATE TABLE BATCH_JOB_EXECUTION (
JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
VERSION BIGINT ,
JOB_INSTANCE_ID BIGINT NOT NULL,
CREATE_TIME TIMESTAMP NOT NULL,
START_TIME TIMESTAMP DEFAULT NULL ,
END_TIME TIMESTAMP DEFAULT NULL ,
STATUS VARCHAR(10) ,
EXIT_CODE VARCHAR(2500) ,
EXIT_MESSAGE VARCHAR(2500) ,
LAST_UPDATED TIMESTAMP,
JOB_CONFIGURATION_LOCATION VARCHAR(2500) ,
constraint JOB_INST_EXEC_FK foreign key (JOB_INSTANCE_ID)
references BATCH_JOB_INSTANCE(JOB_INSTANCE_ID)
) ;
CREATE TABLE BATCH_JOB_EXECUTION_PARAMS (
JOB_EXECUTION_ID BIGINT NOT NULL ,
TYPE_CD VARCHAR(6) NOT NULL ,
KEY_NAME VARCHAR(100) NOT NULL ,
STRING_VAL VARCHAR(250) ,
DATE_VAL TIMESTAMP DEFAULT NULL ,
LONG_VAL BIGINT ,
DOUBLE_VAL DOUBLE ,
IDENTIFYING VARCHAR(1) NOT NULL ,
constraint JOB_EXEC_PARAMS_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
) ;
CREATE TABLE BATCH_STEP_EXECUTION (
STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
VERSION BIGINT NOT NULL,
STEP_NAME VARCHAR(100) NOT NULL,
JOB_EXECUTION_ID BIGINT NOT NULL,
START_TIME TIMESTAMP NOT NULL ,
END_TIME TIMESTAMP DEFAULT NULL ,
STATUS VARCHAR(10) ,
COMMIT_COUNT BIGINT ,
READ_COUNT BIGINT ,
FILTER_COUNT BIGINT ,
WRITE_COUNT BIGINT ,
READ_SKIP_COUNT BIGINT ,
WRITE_SKIP_COUNT BIGINT ,
PROCESS_SKIP_COUNT BIGINT ,
ROLLBACK_COUNT BIGINT ,
EXIT_CODE VARCHAR(2500) ,
EXIT_MESSAGE VARCHAR(2500) ,
LAST_UPDATED TIMESTAMP,
constraint JOB_EXEC_STEP_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
) ;
CREATE TABLE BATCH_STEP_EXECUTION_CONTEXT (
STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY,
SHORT_CONTEXT VARCHAR(2500) NOT NULL,
SERIALIZED_CONTEXT CLOB ,
constraint STEP_EXEC_CTX_FK foreign key (STEP_EXECUTION_ID)
references BATCH_STEP_EXECUTION(STEP_EXECUTION_ID)
) ;
CREATE TABLE BATCH_JOB_EXECUTION_CONTEXT (
JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY,
SHORT_CONTEXT VARCHAR(2500) NOT NULL,
SERIALIZED_CONTEXT CLOB ,
constraint JOB_EXEC_CTX_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
) ;
CREATE SEQUENCE BATCH_STEP_EXECUTION_SEQ START WITH 0 MINVALUE 0 NO CYCLE;
CREATE SEQUENCE BATCH_JOB_EXECUTION_SEQ START WITH 0 MINVALUE 0 NO CYCLE;
CREATE SEQUENCE BATCH_JOB_SEQ START WITH 0 MINVALUE 0 NO CYCLE;

14
spring-batch-core/src/main/sql/hana.properties Normal file
View File

@@ -0,0 +1,14 @@
platform=hana
# SQL language oddities
BIGINT = BIGINT
IDENTITY =
GENERATED = GENERATED BY DEFAULT AS IDENTITY
IFEXISTSBEFORE =
DOUBLE = DOUBLE
BLOB = BLOB
CLOB = CLOB
TIMESTAMP = TIMESTAMP
VARCHAR = VARCHAR
CHAR = VARCHAR
# for generating drop statements...
SEQUENCE = SEQUENCE

3
spring-batch-core/src/main/sql/hana.vpp Normal file
View File

@@ -0,0 +1,3 @@
#macro (sequence $name $value)CREATE SEQUENCE ${name} START WITH ${value} MINVALUE 0 NO CYCLE;
#end
#macro (notnull $name $type)ALTER (${name} ${type} NOT NULL)#end

130
spring-batch-core/src/test/java/org/springframework/batch/core/test/repository/HANAContainer.java Normal file
View File

@@ -0,0 +1,130 @@
/*
* Copyright 2020-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.core.test.repository;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.testcontainers.containers.JdbcDatabaseContainer;
import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.LicenseAcceptance;
import com.github.dockerjava.api.model.Ulimit;
/**
* @author Jonathan Bregler
*/
public class HANAContainer<SELF extends HANAContainer<SELF>> extends JdbcDatabaseContainer<SELF> {
private static final Integer PORT = 39041;
private static final String SYSTEM_USER = "SYSTEM";
private static final String SYSTEM_USER_PASSWORD = "HXEHana1";
public HANAContainer(DockerImageName image) {
super( image );
addExposedPorts( 39013, 39017, 39041, 39042, 39043, 39044, 39045, 1128, 1129, 59013, 59014 );
// create ulimits
Ulimit[] ulimits = new Ulimit[]{ new Ulimit( "nofile", 1048576L, 1048576L ) };
// create sysctls Map.
Map<String, String> sysctls = new HashMap<String, String>();
sysctls.put( "kernel.shmmax", "1073741824" );
sysctls.put( "net.ipv4.ip_local_port_range", "40000 60999" );
// Apply mounts, ulimits and sysctls.
this.withCreateContainerCmdModifier( it -> it.getHostConfig().withUlimits( ulimits ).withSysctls( sysctls ) );
// Arguments for Image.
this.withCommand( "--master-password " + SYSTEM_USER_PASSWORD + " --agree-to-sap-license" );
// Determine if container is ready.
this.waitStrategy = new LogMessageWaitStrategy().withRegEx( ".*Startup finished!*\\s" ).withTimes( 1 )
.withStartupTimeout( Duration.of( 600, ChronoUnit.SECONDS ) );
}
@Override
protected void configure() {
/*
* Enforce that the license is accepted - do not remove. License available at:
* https://www.sap.com/docs/download/cmp/2016/06/sap-hana-express-dev-agmt-and- exhibit.pdf
*/
// If license was not accepted programmatically, check if it was accepted via
// resource file
if ( !getEnvMap().containsKey( "AGREE_TO_SAP_LICENSE" ) ) {
LicenseAcceptance.assertLicenseAccepted( this.getDockerImageName() );
acceptLicense();
}
}
/**
* Accepts the license for the SAP HANA Express container by setting the AGREE_TO_SAP_LICENSE=Y Calling this method
* will automatically accept the license at:
* https://www.sap.com/docs/download/cmp/2016/06/sap-hana-express-dev-agmt-and-exhibit.pdf
*
* @return The container itself with an environment variable accepting the SAP HANA Express license
*/
public SELF acceptLicense() {
addEnv( "AGREE_TO_SAP_LICENSE", "Y" );
return self();
}
@Override
protected Set<Integer> getLivenessCheckPorts() {
return new HashSet<>( Arrays.asList( new Integer[]{ getMappedPort( PORT ) } ) );
}
@Override
protected void waitUntilContainerStarted() {
getWaitStrategy().waitUntilReady( this );
}
@Override
public String getDriverClassName() {
return "com.sap.db.jdbc.Driver";
}
@Override
public String getUsername() {
return SYSTEM_USER;
}
@Override
public String getPassword() {
return SYSTEM_USER_PASSWORD;
}
@Override
public String getTestQueryString() {
return "SELECT 1 FROM SYS.DUMMY";
}
@Override
public String getJdbcUrl() {
return "jdbc:sap://" + getContainerIpAddress() + ":" + getMappedPort( PORT ) + "/";
}
}

106
spring-batch-core/src/test/java/org/springframework/batch/core/test/repository/HANAJobRepositoryIntegrationTests.java Normal file
View File

@@ -0,0 +1,106 @@
/*
* Copyright 2020-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.core.test.repository;
import javax.sql.DataSource;
import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import org.testcontainers.utility.DockerImageName;
import com.sap.db.jdbcext.HanaDataSource;
/**
* @author Jonathan Bregler
*/
@RunWith(SpringRunner.class)
@ContextConfiguration
public class HANAJobRepositoryIntegrationTests {
private static final DockerImageName HANA_IMAGE = DockerImageName.parse( "store/saplabs/hanaexpress:2.00.054.00.20210603.1" );
@ClassRule
public static HANAContainer<?> hana = new HANAContainer<>( HANA_IMAGE ).acceptLicense();
@Autowired
private DataSource dataSource;
@Autowired
private JobLauncher jobLauncher;
@Autowired
private Job job;
@Before
public void setUp() {
ResourceDatabasePopulator databasePopulator = new ResourceDatabasePopulator();
databasePopulator.addScript( new ClassPathResource( "/org/springframework/batch/core/schema-hana.sql" ) );
databasePopulator.execute( this.dataSource );
}
@Test
public void testJobExecution() throws Exception {
// given
JobParameters jobParameters = new JobParametersBuilder().toJobParameters();
// when
JobExecution jobExecution = this.jobLauncher.run( this.job, jobParameters );
// then
Assert.assertNotNull( jobExecution );
Assert.assertEquals( ExitStatus.COMPLETED, jobExecution.getExitStatus() );
}
@Configuration
@EnableBatchProcessing
static class TestConfiguration {
@Bean
public DataSource dataSource() throws Exception {
HanaDataSource dataSource = new HanaDataSource();
dataSource.setUser( hana.getUsername() );
dataSource.setPassword( hana.getPassword() );
dataSource.setUrl( hana.getJdbcUrl() );
return dataSource;
}
@Bean
public Job job(JobBuilderFactory jobs, StepBuilderFactory steps) {
return jobs.get( "job" )
.start( steps.get( "step" ).tasklet( (contribution, chunkContext) -> RepeatStatus.FINISHED ).build() )
.build();
}
}
}