diff --git a/batch/file-ingest/checkstyle.xml b/batch/file-ingest/checkstyle.xml new file mode 100644 index 0000000..5421283 --- /dev/null +++ b/batch/file-ingest/checkstyle.xml @@ -0,0 +1,157 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/batch/file-ingest/pom.xml b/batch/file-ingest/pom.xml new file mode 100644 index 0000000..1e74ed0 --- /dev/null +++ b/batch/file-ingest/pom.xml @@ -0,0 +1,97 @@ + + + 4.0.0 + org.springframework + ingest + 1.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.0.0.M5 + + + 1.8 + 3.7.0 + 1.2.2.RELEASE + 2.0.0.M5 + checkstyle.xml + 2.17 + + + + org.springframework.boot + spring-boot-starter-batch + + + com.h2database + h2 + + + org.springframework.cloud + spring-cloud-task-core + ${spring.cloud.task.version} + + + org.springframework.cloud + spring-cloud-task-batch + ${spring.cloud.task.version} + + + junit + junit + test + + + org.hsqldb + hsqldb + test + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven.compiler.plugin.version} + + ${java.version} + ${java.version} + ${java.version} + ${java.version} + -Xlint:all + + + + + + + repository.spring.milestone + Spring Milestone Repository + http://repo.spring.io/milestone + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + false + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${checkstyle.plugin.version} + + + + diff --git a/batch/file-ingest/src/main/java/org/springframework/ingest/Application.java b/batch/file-ingest/src/main/java/org/springframework/ingest/Application.java new file mode 100644 index 0000000..1cbaf9d --- /dev/null +++ b/batch/file-ingest/src/main/java/org/springframework/ingest/Application.java @@ -0,0 +1,18 @@ +package org.springframework.ingest; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.task.configuration.EnableTask; + +/** + * Main entry point for the ingest sample application. + * + * @author Chris Schaefer + */ +@EnableTask +@SpringBootApplication +public class Application { + public static void main(String[] args) throws Exception { + SpringApplication.run(Application.class, args); + } +} diff --git a/batch/file-ingest/src/main/java/org/springframework/ingest/config/BatchConfiguration.java b/batch/file-ingest/src/main/java/org/springframework/ingest/config/BatchConfiguration.java new file mode 100644 index 0000000..5e03ec0 --- /dev/null +++ b/batch/file-ingest/src/main/java/org/springframework/ingest/config/BatchConfiguration.java @@ -0,0 +1,95 @@ +package org.springframework.ingest.config; + +import javax.sql.DataSource; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemStreamReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; +import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ResourceLoader; + +import org.springframework.ingest.domain.Person; +import org.springframework.ingest.mapper.fieldset.PersonFieldSetMapper; +import org.springframework.ingest.processor.PersonItemProcessor; + +/** + * Class used to configure the batch job related beans. + * + * @author Chris Schaefer + */ +@Configuration +@EnableBatchProcessing +public class BatchConfiguration { + private final DataSource dataSource; + private final ResourceLoader resourceLoader; + private final JobBuilderFactory jobBuilderFactory; + private final StepBuilderFactory stepBuilderFactory; + + @Autowired + public BatchConfiguration(final DataSource dataSource, final JobBuilderFactory jobBuilderFactory, + final StepBuilderFactory stepBuilderFactory, + final ResourceLoader resourceLoader) { + this.dataSource = dataSource; + this.resourceLoader = resourceLoader; + this.jobBuilderFactory = jobBuilderFactory; + this.stepBuilderFactory = stepBuilderFactory; + } + + @Bean + @StepScope + public ItemStreamReader reader(@Value("#{jobParameters['filePath']}") String filePath) throws Exception { + return new FlatFileItemReaderBuilder() + .name("reader") + .resource(resourceLoader.getResource(filePath)) + .delimited() + .names(new String[] {"firstName", "lastName"}) + .fieldSetMapper(new PersonFieldSetMapper()) + .build(); + } + + @Bean + public ItemProcessor processor() { + return new PersonItemProcessor(); + } + + @Bean + public ItemWriter writer() { + return new JdbcBatchItemWriterBuilder() + .beanMapped() + .dataSource(this.dataSource) + .sql("INSERT INTO people (first_name, last_name) VALUES (:firstName, :lastName)") + .build(); + } + + @Bean + public Job ingestJob() throws Exception { + return jobBuilderFactory.get("ingestJob") + .incrementer(new RunIdIncrementer()) + .flow(step1()) + .end() + .build(); + } + + @Bean + public Step step1() throws Exception { + return stepBuilderFactory.get("ingest") + .chunk(10) + .reader(reader(null)) + .processor(processor()) + .writer(writer()) + .build(); + } +} diff --git a/batch/file-ingest/src/main/java/org/springframework/ingest/domain/Person.java b/batch/file-ingest/src/main/java/org/springframework/ingest/domain/Person.java new file mode 100644 index 0000000..edf03d9 --- /dev/null +++ b/batch/file-ingest/src/main/java/org/springframework/ingest/domain/Person.java @@ -0,0 +1,29 @@ +package org.springframework.ingest.domain; + +/** + * Domain object representing data about a Person. + * + * @author Chris Schaefer + */ +public class Person { + private final String firstName; + private final String lastName; + + public Person(final String firstName, final String lastName) { + this.firstName = firstName; + this.lastName = lastName; + } + + public String getFirstName() { + return firstName; + } + + public String getLastName() { + return lastName; + } + + @Override + public String toString() { + return "First name: " + firstName + " , last name: " + lastName; + } +} diff --git a/batch/file-ingest/src/main/java/org/springframework/ingest/mapper/fieldset/PersonFieldSetMapper.java b/batch/file-ingest/src/main/java/org/springframework/ingest/mapper/fieldset/PersonFieldSetMapper.java new file mode 100644 index 0000000..4b3f6cf --- /dev/null +++ b/batch/file-ingest/src/main/java/org/springframework/ingest/mapper/fieldset/PersonFieldSetMapper.java @@ -0,0 +1,21 @@ +package org.springframework.ingest.mapper.fieldset; + +import org.springframework.batch.item.file.mapping.FieldSetMapper; +import org.springframework.batch.item.file.transform.FieldSet; + +import org.springframework.ingest.domain.Person; + +/** + * Maps the provided FieldSet into a Person object. + * + * @author Chris Schaefer + */ +public class PersonFieldSetMapper implements FieldSetMapper { + @Override + public Person mapFieldSet(FieldSet fieldSet) { + String firstName = fieldSet.readString(0); + String lastName = fieldSet.readString(1); + + return new Person(firstName, lastName); + } +} diff --git a/batch/file-ingest/src/main/java/org/springframework/ingest/processor/PersonItemProcessor.java b/batch/file-ingest/src/main/java/org/springframework/ingest/processor/PersonItemProcessor.java new file mode 100644 index 0000000..6d2802c --- /dev/null +++ b/batch/file-ingest/src/main/java/org/springframework/ingest/processor/PersonItemProcessor.java @@ -0,0 +1,29 @@ +package org.springframework.ingest.processor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.batch.item.ItemProcessor; +import org.springframework.ingest.domain.Person; + +/** + * Processes the providing record, transforming the data into + * uppercase characters. + * + * @author Chris Schaefer + */ +public class PersonItemProcessor implements ItemProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(PersonItemProcessor.class); + + @Override + public Person process(Person person) throws Exception { + String firstName = person.getFirstName().toUpperCase(); + String lastName = person.getLastName().toUpperCase(); + + Person processedPerson = new Person(firstName, lastName); + + LOGGER.info("Processed: " + person + " into: " + processedPerson); + + return processedPerson; + } +} diff --git a/batch/file-ingest/src/main/resources/application.properties b/batch/file-ingest/src/main/resources/application.properties new file mode 100644 index 0000000..f5e011e --- /dev/null +++ b/batch/file-ingest/src/main/resources/application.properties @@ -0,0 +1 @@ +spring.application.name=fileIngest diff --git a/batch/file-ingest/src/main/resources/data.csv b/batch/file-ingest/src/main/resources/data.csv new file mode 100644 index 0000000..cead90f --- /dev/null +++ b/batch/file-ingest/src/main/resources/data.csv @@ -0,0 +1,5 @@ +Jill,Doe +Joe,Doe +Justin,Doe +Jane,Doe +John,Doe diff --git a/batch/file-ingest/src/main/resources/schema-all.sql b/batch/file-ingest/src/main/resources/schema-all.sql new file mode 100644 index 0000000..e472ce1 --- /dev/null +++ b/batch/file-ingest/src/main/resources/schema-all.sql @@ -0,0 +1,7 @@ +DROP TABLE people IF EXISTS; + +CREATE TABLE people ( + person_id BIGINT IDENTITY NOT NULL PRIMARY KEY, + first_name VARCHAR(20), + last_name VARCHAR(20) +); diff --git a/batch/file-ingest/src/test/java/org/springframework/ingest/config/BatchConfigurationTests.java b/batch/file-ingest/src/test/java/org/springframework/ingest/config/BatchConfigurationTests.java new file mode 100644 index 0000000..1577977 --- /dev/null +++ b/batch/file-ingest/src/test/java/org/springframework/ingest/config/BatchConfigurationTests.java @@ -0,0 +1,120 @@ +package org.springframework.ingest.config; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ResourceLoader; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseFactory; +import org.springframework.jdbc.datasource.init.DatabasePopulatorUtils; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.util.ClassUtils; + +import java.util.List; +import java.util.Map; + +import javax.annotation.PostConstruct; +import javax.sql.DataSource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * BatchConfiguration test cases + * + * @author Chris Schaefer + */ +public class BatchConfigurationTests { + private static AnnotationConfigApplicationContext context; + + @Before + public void createContext() { + context = new AnnotationConfigApplicationContext(new Class[] { + BatchConfiguration.class, BatchConfigurationTests.DataSourceConfiguration.class }); + } + + @After + public void closeContext() { + context.close(); + } + + @Test + public void testBatchConfigurationSuccess() throws Exception { + JobExecution jobExecution = testJob("classpath:data.csv"); + + assertEquals("Incorrect batch status", BatchStatus.COMPLETED, jobExecution.getStatus()); + assertEquals("Invalid number of step executions", 1, jobExecution.getStepExecutions().size()); + } + + @Test + public void testBatchConfigurationFail() throws Exception { + JobExecution jobExecution = testJob("classpath:missing-data-file.csv"); + + assertEquals("Incorrect batch status", BatchStatus.FAILED, jobExecution.getStatus()); + } + + @Test + public void testBatchDataProcessing() throws Exception { + JobExecution jobExecution = testJob("classpath:data.csv"); + + assertEquals("Incorrect batch status", BatchStatus.COMPLETED, jobExecution.getStatus()); + assertEquals("Invalid number of step executions", 1, jobExecution.getStepExecutions().size()); + + JdbcTemplate jdbcTemplate = new JdbcTemplate(context.getBean(DataSource.class)); + List> peopleList = jdbcTemplate.queryForList("select first_name, last_name from people"); + + assertEquals("Incorrect number of results", 5, peopleList.size()); + + for(Map person : peopleList) { + assertNotNull("Received null person", person); + + String firstName = (String) person.get("first_name"); + assertEquals("Invalid first name: " + firstName, firstName.toUpperCase(), firstName); + + String lastName = (String) person.get("last_name"); + assertEquals("Invalid last name: " + lastName, lastName.toUpperCase(), lastName); + } + } + + private JobExecution testJob(String filePath) throws Exception { + Job job = context.getBean(Job.class); + JobLauncher jobLauncher = context.getBean(JobLauncher.class); + JobParameters jobParameters = new JobParametersBuilder() + .addString("filePath", filePath) + .toJobParameters(); + + return jobLauncher.run(job, jobParameters); + } + + @Configuration + public static class DataSourceConfiguration { + @Autowired + private ResourceLoader resourceLoader; + + @PostConstruct + protected void initialize() { + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(resourceLoader.getResource(ClassUtils.addResourcePathToPackagePath(Step.class, "schema-hsqldb.sql"))); + populator.addScript(resourceLoader.getResource("classpath:schema-all.sql")); + populator.setContinueOnError(true); + DatabasePopulatorUtils.execute(populator, dataSource()); + } + + @Bean + public DataSource dataSource() { + return new EmbeddedDatabaseFactory().getDatabase(); + } + } +} diff --git a/batch/file-ingest/src/test/java/org/springframework/ingest/mapper/fieldset/PersonFieldSetMapperTests.java b/batch/file-ingest/src/test/java/org/springframework/ingest/mapper/fieldset/PersonFieldSetMapperTests.java new file mode 100644 index 0000000..4256de3 --- /dev/null +++ b/batch/file-ingest/src/test/java/org/springframework/ingest/mapper/fieldset/PersonFieldSetMapperTests.java @@ -0,0 +1,36 @@ +package org.springframework.ingest.mapper.fieldset; + +import org.junit.Test; + +import org.springframework.batch.item.file.mapping.FieldSetMapper; +import org.springframework.batch.item.file.transform.DefaultFieldSet; +import org.springframework.batch.item.file.transform.FieldSet; + +import org.springframework.ingest.domain.Person; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; + +/** + * Test cases for PersonFieldSetMapper. + * + * @author Chris Schaefer + */ +public class PersonFieldSetMapperTests { + private static final String[] TOKENS = new String[] { "jane", "doe" }; + private static final String[] NAMES = new String[] { "firstName", "lastName" }; + + @Test + public void testPersonFieldMapping() throws Exception { + FieldSet fieldSet = new DefaultFieldSet(TOKENS, NAMES); + + FieldSetMapper fieldSetMapper = new PersonFieldSetMapper(); + Person person = fieldSetMapper.mapFieldSet(fieldSet); + + assertNotNull("Received null Person", person); + assertNotNull("Received null first name", person.getFirstName()); + assertNotNull("Received null last name", person.getLastName()); + assertEquals("Received wrong first name", TOKENS[0], person.getFirstName()); + assertEquals("Received wrong last name", TOKENS[1], person.getLastName()); + } +} diff --git a/batch/file-ingest/src/test/java/org/springframework/ingest/processor/PersonItemProcessorTests.java b/batch/file-ingest/src/test/java/org/springframework/ingest/processor/PersonItemProcessorTests.java new file mode 100644 index 0000000..1477f58 --- /dev/null +++ b/batch/file-ingest/src/test/java/org/springframework/ingest/processor/PersonItemProcessorTests.java @@ -0,0 +1,35 @@ +package org.springframework.ingest.processor; + +import org.junit.Test; + +import org.springframework.batch.item.ItemProcessor; +import org.springframework.ingest.domain.Person; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; + +/** + * Test cases for PersonItemProcessor. + * + * @author Chris Schaefer + */ +public class PersonItemProcessorTests { + private static final String FIRST_NAME = "jane"; + private static final String LAST_NAME = "doe"; + + @Test + public void testPersonProcessing() throws Exception { + Person person = new Person(FIRST_NAME, LAST_NAME); + + ItemProcessor personItemProcessor = new PersonItemProcessor(); + Person transformedPerson = personItemProcessor.process(person); + + assertNotNull("Received null Person", transformedPerson); + assertNotNull("Received null first name", transformedPerson.getFirstName()); + assertNotNull("Received null last name", transformedPerson.getLastName()); + assertEquals("Invalid first name processing, should be uppercase", + person.getFirstName().toUpperCase(), transformedPerson.getFirstName()); + assertEquals("Invalid last name processing, should be uppercase", + person.getLastName().toUpperCase(), transformedPerson.getLastName()); + } +} diff --git a/src/main/asciidoc/index.adoc b/src/main/asciidoc/index.adoc index d602974..acf95da 100644 --- a/src/main/asciidoc/index.adoc +++ b/src/main/asciidoc/index.adoc @@ -1,5 +1,5 @@ = Spring Cloud Data Flow Samples -Sabby Anandan; David Turanski; Glenn Renfro; Eric Bottard; Mark Pollack; +Sabby Anandan; David Turanski; Glenn Renfro; Eric Bottard; Mark Pollack; Chris Schaefer :doctype: book :toc: :toclevels: 4 diff --git a/src/main/asciidoc/overview.adoc b/src/main/asciidoc/overview.adoc index 1eb7288..7a712ad 100644 --- a/src/main/asciidoc/overview.adoc +++ b/src/main/asciidoc/overview.adoc @@ -12,6 +12,7 @@ include::streaming/custom-apps/celsius-converter-processor/main.adoc[] == Task / Batch include::tasks/simple-batch-job/main.adoc[] +include::tasks/file-ingest/main.adoc[] == Analytics include::analytics/twitter-analytics/main.adoc[] diff --git a/src/main/asciidoc/tasks/file-ingest/main.adoc b/src/main/asciidoc/tasks/file-ingest/main.adoc new file mode 100644 index 0000000..37bb595 --- /dev/null +++ b/src/main/asciidoc/tasks/file-ingest/main.adoc @@ -0,0 +1,121 @@ + +=== Batch File Ingest + +In this demonstration, you will learn how to create a data processing application using http://projects.spring.io/spring-batch/[Spring Batch] which will then be run within http://cloud.spring.io/spring-cloud-dataflow/[Spring Cloud Data Flow]. + +==== Prerequisites + +* A Running Data Flow Server +include::{docs_dir}/local-server.adoc[] + +* A Running Data Flow Shell +include::{docs_dir}/shell.adoc[] + +==== Batch File Ingest Demo Overview + +The source for the demo project is located in the `batch/file-ingest` directory at the top-level of this repository. The sample is a Spring Boot application that demonstrates how to read data from a flat file, perform processing on the records, and store the transformed data into a database using Spring Batch. + +The key classes for creating the batch job are: + +* `BatchConfiguration.java` - this is where we define our batch job, the step and components that are used read, process, and write our data. In the sample we use a `FlatFileItemReader` which reads a delimited file, a custom `PersonItemProcessor` to transform the data, and a `JdbcBatchItemWriter` to write our data to a database. + +* `Person.java` - the domain object representing the data we are reading and processing in our batch job. The sample data contains records made up of a persons first and last name. + +* `PersonItemProcessor.java` - this class is an `ItemProcessor` implementation which receives records after they have been read and before they are written. This allows us to transform the data between these two steps. In our sample `ItemProcessor` implementation, we simply transform the first and last name of each `Person` to uppercase characters. + +* `Application.java` - the main entry point into the Spring Boot application which is used to launch the batch job + +Resource files are included to set up the database and provide sample data: + +* `schema-all.sql` - this is the database schema that will be created when the application starts up. In this sample, an in-memory database is created on start up and destroyed when the application exits. + +* `data.csv` - sample data file containing person records used in the demo + + +==== Building and Running the Demo + +. Build the demo JAR ++ +``` +$ mvn clean package +``` + ++ + +. Register the task ++ +``` +dataflow:>app register --name fileIngest --type task --uri file:////path/to/target/ingest-X.X.X.jar +Successfully registered application 'task:fileIngest' +dataflow:> +``` + ++ + +. Create the task ++ +``` +dataflow:>task create fileIngestTask --definition fileIngest +Created new task 'fileIngestTask' +dataflow:> +``` + ++ + +. Launch the task ++ +``` +dataflow:>task launch fileIngestTask --arguments "filePath=classpath:data.csv" +Launched task 'fileIngestTask' +dataflow:> +``` + ++ + +. Inspect logs ++ +The log file path for the launched task can be found in the local server output, for example: ++ + +[source,console,options=nowrap] +---- +2017-10-27 14:58:18.112 INFO 19485 --- [nio-9393-exec-6] o.s.c.d.spi.local.LocalTaskLauncher : launching task fileIngestTask-8932f73d-f17a-4bba-b44d-3fd9df042ac0 + Logs will be in /var/folders/6x/tgtx9xbn0x16xq2sx1j2rld80000gn/T/spring-cloud-dataflow-983191515779755562/fileIngestTask-1509130698071/fileIngestTask-8932f73d-f17a-4bba-b44d-3fd9df042ac0 +---- + +. Verify Task execution details + ++ + +[source,console,options=nowrap] +---- +dataflow:>task execution list +╔══════════════╤══╤════════════════════════════╤════════════════════════════╤═════════╗ +║ Task Name │ID│ Start Time │ End Time │Exit Code║ +╠══════════════╪══╪════════════════════════════╪════════════════════════════╪═════════╣ +║fileIngestTask│1 │Fri Oct 27 14:58:20 EDT 2017│Fri Oct 27 14:58:20 EDT 2017│0 ║ +╚══════════════╧══╧════════════════════════════╧════════════════════════════╧═════════╝ +---- + +. Verify Job execution details + ++ + +[source,console,options=nowrap] +---- +dataflow:>job execution list +╔═══╤═══════╤═════════╤════════════════════════════╤═════════════════════╤══════════════════╗ +║ID │Task ID│Job Name │ Start Time │Step Execution Count │Definition Status ║ +╠═══╪═══════╪═════════╪════════════════════════════╪═════════════════════╪══════════════════╣ +║1 │1 │ingestJob│Fri Oct 27 14:58:20 EDT 2017│1 │Created ║ +╚═══╧═══════╧═════════╧════════════════════════════╧═════════════════════╧══════════════════╝ +---- + + +==== Summary + +In this sample, you have learned: + +* How to create a data processing batch job application +* How to register and orchestrate Spring Batch jobs in Spring Cloud Data Flow +* How to verify status via logs and shell commands