SpringCloud/spring-cloud-dataflow-samples
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Main File Ingest Demo written

Browse Source 
* Add Concurrent Execution Limit
* Add Avoid Duplicate Processing
* Update link
* Add snapshot repo
* Update copyright year
* Finish Metadata Store sample and reduce number of split files to 20
* Add Cloud Foundry MetadataStore
This commit is contained in:
David Turanski
2018-10-26 15:09:59 -04:00
committed by Chris Schaefer
parent 4a8fbbd723
commit 699f7856e4
72 changed files with 11783 additions and 2215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
batch/file-ingest/src/main/java/io/spring/cloud/dataflow/ingest/Application.java
View File

@@ -18,17 +18,16 @@ package io.spring.cloud.dataflow.ingest;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.task.configuration.EnableTask;
/**
* Main entry point for the ingest sample application.
*
* @author Chris Schaefer
* @author David Turanski
*/
@EnableTask
@SpringBootApplication
public class Application {
public static void main(String[] args) throws Exception {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}

23
batch/file-ingest/src/main/java/io/spring/cloud/dataflow/ingest/config/BatchConfiguration.java
View File

@@ -33,7 +33,6 @@ import org.springframework.batch.item.ItemStreamReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
@@ -44,19 +43,24 @@ import org.springframework.core.io.ResourceLoader;
* Class used to configure the batch job related beans.
*
* @author Chris Schaefer
* @author David Turanski
*/
@Configuration
@EnableBatchProcessing
public class BatchConfiguration {
private final DataSource dataSource;
private final ResourceLoader resourceLoader;
private final JobBuilderFactory jobBuilderFactory;
private final StepBuilderFactory stepBuilderFactory;
@Autowired
public BatchConfiguration(final DataSource dataSource, final JobBuilderFactory jobBuilderFactory,
final StepBuilderFactory stepBuilderFactory,
final ResourceLoader resourceLoader) {
final StepBuilderFactory stepBuilderFactory,
final ResourceLoader resourceLoader) {
this.dataSource = dataSource;
this.resourceLoader = resourceLoader;
this.jobBuilderFactory = jobBuilderFactory;
@@ -65,12 +69,17 @@ public class BatchConfiguration {
@Bean
@StepScope
public ItemStreamReader<Person> reader(@Value("#{jobParameters['filePath']}") String filePath) throws Exception {
public ItemStreamReader<Person> reader(@Value("#{jobParameters['localFilePath']}") String filePath) {
if (!filePath.matches("[a-z]+:.*")) {
filePath = "file:" + filePath;
}
return new FlatFileItemReaderBuilder<Person>()
.name("reader")
.resource(resourceLoader.getResource(filePath))
.delimited()
.names(new String[] {"firstName", "lastName"})
.names(new String[] { "firstName", "lastName" })
.fieldSetMapper(new PersonFieldSetMapper())
.build();
}
@@ -90,7 +99,7 @@ public class BatchConfiguration {
}
@Bean
public Job ingestJob() throws Exception {
public Job ingestJob() {
return jobBuilderFactory.get("ingestJob")
.incrementer(new RunIdIncrementer())
.flow(step1())
@@ -99,7 +108,7 @@ public class BatchConfiguration {
}
@Bean
public Step step1() throws Exception {
public Step step1() {
return stepBuilderFactory.get("ingest")
.<Person, Person>chunk(10)
.reader(reader(null))

1
batch/file-ingest/src/main/resources/application.properties
View File

@@ -1 +1,2 @@
spring.application.name=fileIngest
spring.datasource.initialization-mode=always

9
batch/file-ingest/src/main/resources/schema-all.sql
View File

@@ -1,7 +1,6 @@
DROP TABLE people IF EXISTS;
CREATE TABLE people (
person_id BIGINT IDENTITY NOT NULL PRIMARY KEY,
CREATE TABLE IF NOT EXISTS people (
person_id BIGINT NOT NULL AUTO_INCREMENT,
first_name VARCHAR(20),
last_name VARCHAR(20)
last_name VARCHAR(20),
PRIMARY KEY (person_id)
);

101
batch/file-ingest/src/test/java/io/spring/cloud/dataflow/ingest/BatchApplicationTests.java Normal file
View File

@@ -0,0 +1,101 @@
/*
* Copyright 2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.spring.cloud.dataflow.ingest;
import java.util.List;
import java.util.Map;
import io.spring.cloud.dataflow.ingest.config.BatchConfiguration;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.test.JobLauncherTestUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
* BatchConfiguration test cases
*
* @author Chris Schaefer
* @author David Turanski
*/
@RunWith(SpringRunner.class)
@ContextConfiguration(classes = { BatchConfiguration.class, BatchApplicationTests.BatchTestConfiguration.class })
public class BatchApplicationTests {
@Autowired
private JobLauncherTestUtils jobLauncherTestUtils;
@Autowired
private JdbcTemplate jdbcTemplate;
@Test
public void testBatchConfigurationFail() throws Exception {
BatchStatus status = jobLauncherTestUtils.launchJob(new JobParametersBuilder().addString(
"localFilePath", "classpath:missing-data.csv").toJobParameters()).getStatus();
assertEquals("Incorrect batch status", BatchStatus.FAILED, status);
}
@Test
public void testBatchDataProcessing() throws Exception {
JobExecution jobExecution = jobLauncherTestUtils.launchJob(new JobParametersBuilder().addString(
"localFilePath", "classpath:data.csv").toJobParameters());
assertEquals("Incorrect batch status", BatchStatus.COMPLETED, jobExecution.getStatus());
assertEquals("Invalid number of step executions", 1, jobExecution.getStepExecutions().size());
List<Map<String, Object>> peopleList = jdbcTemplate.queryForList(
"select first_name, last_name from people");
assertEquals("Incorrect number of results", 5, peopleList.size());
for (Map<String, Object> person : peopleList) {
assertNotNull("Received null person", person);
String firstName = (String) person.get("first_name");
assertEquals("Invalid first name: " + firstName, firstName.toUpperCase(), firstName);
String lastName = (String) person.get("last_name");
assertEquals("Invalid last name: " + lastName, lastName.toUpperCase(), lastName);
}
}
@Configuration
@EnableAutoConfiguration
public static class BatchTestConfiguration {
@Bean
public JobLauncherTestUtils jobLauncherTestUtils() {
return new JobLauncherTestUtils();
}
}
}

136
batch/file-ingest/src/test/java/io/spring/cloud/dataflow/ingest/config/BatchConfigurationTests.java
View File

@@ -1,136 +0,0 @@
/*
* Copyright 2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.spring.cloud.dataflow.ingest.config;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ResourceLoader;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseFactory;
import org.springframework.jdbc.datasource.init.DatabasePopulatorUtils;
import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator;
import org.springframework.util.ClassUtils;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.sql.DataSource;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
* BatchConfiguration test cases
*
* @author Chris Schaefer
*/
public class BatchConfigurationTests {
private static AnnotationConfigApplicationContext context;
@Before
public void createContext() {
context = new AnnotationConfigApplicationContext(new Class[] {
BatchConfiguration.class, BatchConfigurationTests.DataSourceConfiguration.class });
}
@After
public void closeContext() {
context.close();
}
@Test
public void testBatchConfigurationSuccess() throws Exception {
JobExecution jobExecution = testJob("classpath:data.csv");
assertEquals("Incorrect batch status", BatchStatus.COMPLETED, jobExecution.getStatus());
assertEquals("Invalid number of step executions", 1, jobExecution.getStepExecutions().size());
}
@Test
public void testBatchConfigurationFail() throws Exception {
JobExecution jobExecution = testJob("classpath:missing-data-file.csv");
assertEquals("Incorrect batch status", BatchStatus.FAILED, jobExecution.getStatus());
}
@Test
public void testBatchDataProcessing() throws Exception {
JobExecution jobExecution = testJob("classpath:data.csv");
assertEquals("Incorrect batch status", BatchStatus.COMPLETED, jobExecution.getStatus());
assertEquals("Invalid number of step executions", 1, jobExecution.getStepExecutions().size());
JdbcTemplate jdbcTemplate = new JdbcTemplate(context.getBean(DataSource.class));
List<Map<String, Object>> peopleList = jdbcTemplate.queryForList("select first_name, last_name from people");
assertEquals("Incorrect number of results", 5, peopleList.size());
for(Map<String, Object> person : peopleList) {
assertNotNull("Received null person", person);
String firstName = (String) person.get("first_name");
assertEquals("Invalid first name: " + firstName, firstName.toUpperCase(), firstName);
String lastName = (String) person.get("last_name");
assertEquals("Invalid last name: " + lastName, lastName.toUpperCase(), lastName);
}
}
private JobExecution testJob(String filePath) throws Exception {
Job job = context.getBean(Job.class);
JobLauncher jobLauncher = context.getBean(JobLauncher.class);
JobParameters jobParameters = new JobParametersBuilder()
.addString("filePath", filePath)
.toJobParameters();
return jobLauncher.run(job, jobParameters);
}
@Configuration
public static class DataSourceConfiguration {
@Autowired
private ResourceLoader resourceLoader;
@PostConstruct
protected void initialize() {
ResourceDatabasePopulator populator = new ResourceDatabasePopulator();
populator.addScript(resourceLoader.getResource(ClassUtils.addResourcePathToPackagePath(Step.class, "schema-hsqldb.sql")));
populator.addScript(resourceLoader.getResource("classpath:schema-all.sql"));
populator.setContinueOnError(true);
DatabasePopulatorUtils.execute(populator, dataSource());
}
@Bean
public DataSource dataSource() {
return new EmbeddedDatabaseFactory().getDatabase();
}
}
}