Create basic batch ingest sample
* Read from a file, apply processing, write to database * Hook in Spring Cloud Task * Ensure job can be registered and executed through data flow * Create README documenting build steps and how to register / launch the job as a task via Data Flow shell
This commit is contained in:
Chris Schaefer
committed by
David Turanski
David Turanski
parent
02fc1d308a
commit
b68aaf2487
157
batch/file-ingest/checkstyle.xml
Normal file
157
batch/file-ingest/checkstyle.xml
Normal file
@@ -0,0 +1,157 @@
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.2//EN" "http://www.puppycrawl.com/dtds/configuration_1_2.dtd">
|
||||
<module name="com.puppycrawl.tools.checkstyle.Checker">
|
||||
<!-- Root Checks -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.NewlineAtEndOfFileCheck">
|
||||
<property name="lineSeparator" value="lf"/>
|
||||
</module>
|
||||
|
||||
<!-- TreeWalker Checks -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.TreeWalker">
|
||||
<!-- Annotations -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.annotation.AnnotationUseStyleCheck">
|
||||
<property name="elementStyle" value="compact" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.annotation.MissingOverrideCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.annotation.PackageAnnotationCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.annotation.AnnotationLocationCheck">
|
||||
<property name="allowSamelineSingleParameterlessAnnotation"
|
||||
value="false" />
|
||||
</module>
|
||||
|
||||
<!-- Block Checks -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.blocks.EmptyBlockCheck">
|
||||
<property name="option" value="text" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.blocks.LeftCurlyCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.blocks.RightCurlyCheck">
|
||||
<property name="option" value="alone" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.blocks.NeedBracesCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.blocks.AvoidNestedBlocksCheck" />
|
||||
|
||||
<!-- Class Design -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.design.FinalClassCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.design.InterfaceIsTypeCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.design.HideUtilityClassConstructorCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.design.MutableExceptionCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.design.InnerTypeLastCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.design.OneTopLevelClassCheck" />
|
||||
|
||||
<!-- Coding -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.CovariantEqualsCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.EmptyStatementCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.EqualsHashCodeCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.InnerAssignmentCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.SimplifyBooleanExpressionCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.SimplifyBooleanReturnCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.StringLiteralEqualityCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.NestedForDepthCheck">
|
||||
<property name="max" value="3" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.NestedIfDepthCheck">
|
||||
<property name="max" value="3" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.NestedTryDepthCheck">
|
||||
<property name="max" value="3" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.MultipleVariableDeclarationsCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.coding.OneStatementPerLineCheck" />
|
||||
|
||||
<!-- Imports -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.imports.AvoidStarImportCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.imports.AvoidStaticImportCheck">
|
||||
<property name="excludes"
|
||||
value="io.restassured.RestAssured.*, org.assertj.core.api.Assertions.*, org.junit.Assert.*, org.junit.Assume.*, org.junit.internal.matchers.ThrowableMessageMatcher.*, org.hamcrest.CoreMatchers.*, org.hamcrest.Matchers.*, org.springframework.boot.configurationprocessor.ConfigurationMetadataMatchers.*, org.springframework.boot.configurationprocessor.TestCompiler.*, org.springframework.boot.test.autoconfigure.AutoConfigurationImportedCondition.*, org.mockito.Mockito.*, org.mockito.BDDMockito.*, org.mockito.ArgumentMatchers.*, org.mockito.Matchers.*, org.springframework.restdocs.hypermedia.HypermediaDocumentation.*, org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.*, org.springframework.restdocs.operation.preprocess.Preprocessors.*, org.springframework.restdocs.restassured3.operation.preprocess.RestAssuredPreprocessors.*, org.springframework.restdocs.restassured3.RestAssuredRestDocumentation.*, org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*, org.springframework.test.web.servlet.result.MockMvcResultMatchers.*, org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.*, org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.*, org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo, org.springframework.test.web.client.ExpectedCount.*, org.springframework.test.web.client.match.MockRestRequestMatchers.*, org.springframework.test.web.client.response.MockRestResponseCreators.*" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.imports.IllegalImportCheck" >
|
||||
<property name="illegalPkgs" value="com.google.common"/>
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.imports.RedundantImportCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.imports.UnusedImportsCheck">
|
||||
<property name="processJavadoc" value="true" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.imports.ImportOrderCheck">
|
||||
<property name="groups" value="java,/^javax?\./,*,org.springframework" />
|
||||
<property name="ordered" value="true" />
|
||||
<property name="separated" value="true" />
|
||||
<property name="option" value="bottom" />
|
||||
<property name="sortStaticImportsAlphabetically" value="true" />
|
||||
</module>
|
||||
|
||||
<!-- Javadoc Comments -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocTypeCheck">
|
||||
<property name="scope" value="package"/>
|
||||
<property name="authorFormat" value=".+\s.+"/>
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocMethodCheck">
|
||||
<property name="allowMissingJavadoc" value="true" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocVariableCheck">
|
||||
<property name="scope" value="public"/>
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocStyleCheck">
|
||||
<property name="checkEmptyJavadoc" value="true"/>
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.javadoc.NonEmptyAtclauseDescriptionCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocTagContinuationIndentationCheck">
|
||||
<property name="offset" value="0"/>
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.javadoc.AtclauseOrderCheck">
|
||||
<property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF"/>
|
||||
<property name="tagOrder" value="@param, @author, @since, @see, @version, @serial, @deprecated"/>
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.javadoc.AtclauseOrderCheck">
|
||||
<property name="target" value="METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
|
||||
<property name="tagOrder" value="@param, @return, @throws, @since, @deprecated, @see"/>
|
||||
</module>
|
||||
|
||||
<!-- Miscellaneous -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.indentation.CommentsIndentationCheck">
|
||||
<property name="tokens" value="BLOCK_COMMENT_BEGIN"/>
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.UpperEllCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.ArrayTypeStyleCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.OuterTypeFilenameCheck" />
|
||||
|
||||
<!-- Modifiers -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.modifier.RedundantModifierCheck" />
|
||||
|
||||
<!-- Regexp -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.regexp.RegexpSinglelineJavaCheck">
|
||||
<property name="format" value="^\t* +\t*\S" />
|
||||
<property name="message"
|
||||
value="Line has leading space characters; indentation should be performed with tabs only." />
|
||||
<property name="ignoreComments" value="true" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.regexp.RegexpSinglelineJavaCheck">
|
||||
<property name="maximum" value="0"/>
|
||||
<property name="format" value="org\.mockito\.Mockito\.(when|doThrow|doAnswer)" />
|
||||
<property name="message" value="Please use BDDMockito imports." />
|
||||
<property name="ignoreComments" value="true" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.regexp.RegexpSinglelineJavaCheck">
|
||||
<property name="maximum" value="0"/>
|
||||
<property name="format" value="org\.junit\.Assert\.assert" />
|
||||
<property name="message" value="Please use AssertJ imports." />
|
||||
<property name="ignoreComments" value="true" />
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.regexp.RegexpCheck">
|
||||
<property name="format" value="[ \t]+$" />
|
||||
<property name="illegalPattern" value="true" />
|
||||
<property name="message" value="Trailing whitespace" />
|
||||
</module>
|
||||
|
||||
<!-- Whitespace -->
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.whitespace.GenericWhitespaceCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.whitespace.MethodParamPadCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.whitespace.NoWhitespaceAfterCheck" >
|
||||
<property name="tokens" value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS, UNARY_PLUS, ARRAY_DECLARATOR"/>
|
||||
</module>
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.whitespace.NoWhitespaceBeforeCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.whitespace.ParenPadCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.whitespace.TypecastParenPadCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.whitespace.WhitespaceAfterCheck" />
|
||||
<module name="com.puppycrawl.tools.checkstyle.checks.whitespace.WhitespaceAroundCheck" />
|
||||
</module>
|
||||
</module>
|
||||
97
batch/file-ingest/pom.xml
Normal file
97
batch/file-ingest/pom.xml
Normal file
@@ -0,0 +1,97 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>ingest</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.0.0.M5</version>
|
||||
</parent>
|
||||
<properties>
|
||||
<java.version>1.8</java.version>
|
||||
<maven.compiler.plugin.version>3.7.0</maven.compiler.plugin.version>
|
||||
<spring.cloud.task.version>1.2.2.RELEASE</spring.cloud.task.version>
|
||||
<spring.batch.starter.version>2.0.0.M5</spring.batch.starter.version>
|
||||
<checkstyle.config.location>checkstyle.xml</checkstyle.config.location>
|
||||
<checkstyle.plugin.version>2.17</checkstyle.plugin.version>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-batch</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-task-core</artifactId>
|
||||
<version>${spring.cloud.task.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-task-batch</artifactId>
|
||||
<version>${spring.cloud.task.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.hsqldb</groupId>
|
||||
<artifactId>hsqldb</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>${maven.compiler.plugin.version}</version>
|
||||
<configuration>
|
||||
<source>${java.version}</source>
|
||||
<target>${java.version}</target>
|
||||
<testSource>${java.version}</testSource>
|
||||
<testTarget>${java.version}</testTarget>
|
||||
<compilerArgument>-Xlint:all</compilerArgument>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>repository.spring.milestone</id>
|
||||
<name>Spring Milestone Repository</name>
|
||||
<url>http://repo.spring.io/milestone</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/milestone</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
<reporting>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<version>${checkstyle.plugin.version}</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</reporting>
|
||||
</project>
|
||||
@@ -0,0 +1,18 @@
|
||||
package org.springframework.ingest;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.cloud.task.configuration.EnableTask;
|
||||
|
||||
/**
|
||||
* Main entry point for the ingest sample application.
|
||||
*
|
||||
* @author Chris Schaefer
|
||||
*/
|
||||
@EnableTask
|
||||
@SpringBootApplication
|
||||
public class Application {
|
||||
public static void main(String[] args) throws Exception {
|
||||
SpringApplication.run(Application.class, args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package org.springframework.ingest.config;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.batch.core.Job;
|
||||
import org.springframework.batch.core.Step;
|
||||
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
|
||||
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
|
||||
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
|
||||
import org.springframework.batch.core.configuration.annotation.StepScope;
|
||||
import org.springframework.batch.core.launch.support.RunIdIncrementer;
|
||||
import org.springframework.batch.item.ItemProcessor;
|
||||
import org.springframework.batch.item.ItemStreamReader;
|
||||
import org.springframework.batch.item.ItemWriter;
|
||||
import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
|
||||
import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.core.io.ResourceLoader;
|
||||
|
||||
import org.springframework.ingest.domain.Person;
|
||||
import org.springframework.ingest.mapper.fieldset.PersonFieldSetMapper;
|
||||
import org.springframework.ingest.processor.PersonItemProcessor;
|
||||
|
||||
/**
|
||||
* Class used to configure the batch job related beans.
|
||||
*
|
||||
* @author Chris Schaefer
|
||||
*/
|
||||
@Configuration
|
||||
@EnableBatchProcessing
|
||||
public class BatchConfiguration {
|
||||
private final DataSource dataSource;
|
||||
private final ResourceLoader resourceLoader;
|
||||
private final JobBuilderFactory jobBuilderFactory;
|
||||
private final StepBuilderFactory stepBuilderFactory;
|
||||
|
||||
@Autowired
|
||||
public BatchConfiguration(final DataSource dataSource, final JobBuilderFactory jobBuilderFactory,
|
||||
final StepBuilderFactory stepBuilderFactory,
|
||||
final ResourceLoader resourceLoader) {
|
||||
this.dataSource = dataSource;
|
||||
this.resourceLoader = resourceLoader;
|
||||
this.jobBuilderFactory = jobBuilderFactory;
|
||||
this.stepBuilderFactory = stepBuilderFactory;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@StepScope
|
||||
public ItemStreamReader<Person> reader(@Value("#{jobParameters['filePath']}") String filePath) throws Exception {
|
||||
return new FlatFileItemReaderBuilder<Person>()
|
||||
.name("reader")
|
||||
.resource(resourceLoader.getResource(filePath))
|
||||
.delimited()
|
||||
.names(new String[] {"firstName", "lastName"})
|
||||
.fieldSetMapper(new PersonFieldSetMapper())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ItemProcessor<Person, Person> processor() {
|
||||
return new PersonItemProcessor();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ItemWriter<Person> writer() {
|
||||
return new JdbcBatchItemWriterBuilder<Person>()
|
||||
.beanMapped()
|
||||
.dataSource(this.dataSource)
|
||||
.sql("INSERT INTO people (first_name, last_name) VALUES (:firstName, :lastName)")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Job ingestJob() throws Exception {
|
||||
return jobBuilderFactory.get("ingestJob")
|
||||
.incrementer(new RunIdIncrementer())
|
||||
.flow(step1())
|
||||
.end()
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Step step1() throws Exception {
|
||||
return stepBuilderFactory.get("ingest")
|
||||
.<Person, Person>chunk(10)
|
||||
.reader(reader(null))
|
||||
.processor(processor())
|
||||
.writer(writer())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package org.springframework.ingest.domain;
|
||||
|
||||
/**
|
||||
* Domain object representing data about a Person.
|
||||
*
|
||||
* @author Chris Schaefer
|
||||
*/
|
||||
public class Person {
|
||||
private final String firstName;
|
||||
private final String lastName;
|
||||
|
||||
public Person(final String firstName, final String lastName) {
|
||||
this.firstName = firstName;
|
||||
this.lastName = lastName;
|
||||
}
|
||||
|
||||
public String getFirstName() {
|
||||
return firstName;
|
||||
}
|
||||
|
||||
public String getLastName() {
|
||||
return lastName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "First name: " + firstName + " , last name: " + lastName;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package org.springframework.ingest.mapper.fieldset;
|
||||
|
||||
import org.springframework.batch.item.file.mapping.FieldSetMapper;
|
||||
import org.springframework.batch.item.file.transform.FieldSet;
|
||||
|
||||
import org.springframework.ingest.domain.Person;
|
||||
|
||||
/**
|
||||
* Maps the provided FieldSet into a Person object.
|
||||
*
|
||||
* @author Chris Schaefer
|
||||
*/
|
||||
public class PersonFieldSetMapper implements FieldSetMapper<Person> {
|
||||
@Override
|
||||
public Person mapFieldSet(FieldSet fieldSet) {
|
||||
String firstName = fieldSet.readString(0);
|
||||
String lastName = fieldSet.readString(1);
|
||||
|
||||
return new Person(firstName, lastName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package org.springframework.ingest.processor;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.springframework.batch.item.ItemProcessor;
|
||||
import org.springframework.ingest.domain.Person;
|
||||
|
||||
/**
|
||||
* Processes the providing record, transforming the data into
|
||||
* uppercase characters.
|
||||
*
|
||||
* @author Chris Schaefer
|
||||
*/
|
||||
public class PersonItemProcessor implements ItemProcessor<Person, Person> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PersonItemProcessor.class);
|
||||
|
||||
@Override
|
||||
public Person process(Person person) throws Exception {
|
||||
String firstName = person.getFirstName().toUpperCase();
|
||||
String lastName = person.getLastName().toUpperCase();
|
||||
|
||||
Person processedPerson = new Person(firstName, lastName);
|
||||
|
||||
LOGGER.info("Processed: " + person + " into: " + processedPerson);
|
||||
|
||||
return processedPerson;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
spring.application.name=fileIngest
|
||||
5
batch/file-ingest/src/main/resources/data.csv
Normal file
5
batch/file-ingest/src/main/resources/data.csv
Normal file
@@ -0,0 +1,5 @@
|
||||
Jill,Doe
|
||||
Joe,Doe
|
||||
Justin,Doe
|
||||
Jane,Doe
|
||||
John,Doe
|
||||
|
7
batch/file-ingest/src/main/resources/schema-all.sql
Normal file
7
batch/file-ingest/src/main/resources/schema-all.sql
Normal file
@@ -0,0 +1,7 @@
|
||||
DROP TABLE people IF EXISTS;
|
||||
|
||||
CREATE TABLE people (
|
||||
person_id BIGINT IDENTITY NOT NULL PRIMARY KEY,
|
||||
first_name VARCHAR(20),
|
||||
last_name VARCHAR(20)
|
||||
);
|
||||
@@ -0,0 +1,120 @@
|
||||
package org.springframework.ingest.config;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.springframework.batch.core.BatchStatus;
|
||||
import org.springframework.batch.core.Job;
|
||||
import org.springframework.batch.core.JobExecution;
|
||||
import org.springframework.batch.core.JobParameters;
|
||||
import org.springframework.batch.core.JobParametersBuilder;
|
||||
import org.springframework.batch.core.Step;
|
||||
import org.springframework.batch.core.launch.JobLauncher;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.core.io.ResourceLoader;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseFactory;
|
||||
import org.springframework.jdbc.datasource.init.DatabasePopulatorUtils;
|
||||
import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator;
|
||||
import org.springframework.util.ClassUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
|
||||
/**
|
||||
* BatchConfiguration test cases
|
||||
*
|
||||
* @author Chris Schaefer
|
||||
*/
|
||||
public class BatchConfigurationTests {
|
||||
private static AnnotationConfigApplicationContext context;
|
||||
|
||||
@Before
|
||||
public void createContext() {
|
||||
context = new AnnotationConfigApplicationContext(new Class[] {
|
||||
BatchConfiguration.class, BatchConfigurationTests.DataSourceConfiguration.class });
|
||||
}
|
||||
|
||||
@After
|
||||
public void closeContext() {
|
||||
context.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBatchConfigurationSuccess() throws Exception {
|
||||
JobExecution jobExecution = testJob("classpath:data.csv");
|
||||
|
||||
assertEquals("Incorrect batch status", BatchStatus.COMPLETED, jobExecution.getStatus());
|
||||
assertEquals("Invalid number of step executions", 1, jobExecution.getStepExecutions().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBatchConfigurationFail() throws Exception {
|
||||
JobExecution jobExecution = testJob("classpath:missing-data-file.csv");
|
||||
|
||||
assertEquals("Incorrect batch status", BatchStatus.FAILED, jobExecution.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBatchDataProcessing() throws Exception {
|
||||
JobExecution jobExecution = testJob("classpath:data.csv");
|
||||
|
||||
assertEquals("Incorrect batch status", BatchStatus.COMPLETED, jobExecution.getStatus());
|
||||
assertEquals("Invalid number of step executions", 1, jobExecution.getStepExecutions().size());
|
||||
|
||||
JdbcTemplate jdbcTemplate = new JdbcTemplate(context.getBean(DataSource.class));
|
||||
List<Map<String, Object>> peopleList = jdbcTemplate.queryForList("select first_name, last_name from people");
|
||||
|
||||
assertEquals("Incorrect number of results", 5, peopleList.size());
|
||||
|
||||
for(Map<String, Object> person : peopleList) {
|
||||
assertNotNull("Received null person", person);
|
||||
|
||||
String firstName = (String) person.get("first_name");
|
||||
assertEquals("Invalid first name: " + firstName, firstName.toUpperCase(), firstName);
|
||||
|
||||
String lastName = (String) person.get("last_name");
|
||||
assertEquals("Invalid last name: " + lastName, lastName.toUpperCase(), lastName);
|
||||
}
|
||||
}
|
||||
|
||||
private JobExecution testJob(String filePath) throws Exception {
|
||||
Job job = context.getBean(Job.class);
|
||||
JobLauncher jobLauncher = context.getBean(JobLauncher.class);
|
||||
JobParameters jobParameters = new JobParametersBuilder()
|
||||
.addString("filePath", filePath)
|
||||
.toJobParameters();
|
||||
|
||||
return jobLauncher.run(job, jobParameters);
|
||||
}
|
||||
|
||||
@Configuration
|
||||
public static class DataSourceConfiguration {
|
||||
@Autowired
|
||||
private ResourceLoader resourceLoader;
|
||||
|
||||
@PostConstruct
|
||||
protected void initialize() {
|
||||
ResourceDatabasePopulator populator = new ResourceDatabasePopulator();
|
||||
populator.addScript(resourceLoader.getResource(ClassUtils.addResourcePathToPackagePath(Step.class, "schema-hsqldb.sql")));
|
||||
populator.addScript(resourceLoader.getResource("classpath:schema-all.sql"));
|
||||
populator.setContinueOnError(true);
|
||||
DatabasePopulatorUtils.execute(populator, dataSource());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public DataSource dataSource() {
|
||||
return new EmbeddedDatabaseFactory().getDatabase();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package org.springframework.ingest.mapper.fieldset;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import org.springframework.batch.item.file.mapping.FieldSetMapper;
|
||||
import org.springframework.batch.item.file.transform.DefaultFieldSet;
|
||||
import org.springframework.batch.item.file.transform.FieldSet;
|
||||
|
||||
import org.springframework.ingest.domain.Person;
|
||||
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* Test cases for PersonFieldSetMapper.
|
||||
*
|
||||
* @author Chris Schaefer
|
||||
*/
|
||||
public class PersonFieldSetMapperTests {
|
||||
private static final String[] TOKENS = new String[] { "jane", "doe" };
|
||||
private static final String[] NAMES = new String[] { "firstName", "lastName" };
|
||||
|
||||
@Test
|
||||
public void testPersonFieldMapping() throws Exception {
|
||||
FieldSet fieldSet = new DefaultFieldSet(TOKENS, NAMES);
|
||||
|
||||
FieldSetMapper<Person> fieldSetMapper = new PersonFieldSetMapper();
|
||||
Person person = fieldSetMapper.mapFieldSet(fieldSet);
|
||||
|
||||
assertNotNull("Received null Person", person);
|
||||
assertNotNull("Received null first name", person.getFirstName());
|
||||
assertNotNull("Received null last name", person.getLastName());
|
||||
assertEquals("Received wrong first name", TOKENS[0], person.getFirstName());
|
||||
assertEquals("Received wrong last name", TOKENS[1], person.getLastName());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package org.springframework.ingest.processor;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import org.springframework.batch.item.ItemProcessor;
|
||||
import org.springframework.ingest.domain.Person;
|
||||
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* Test cases for PersonItemProcessor.
|
||||
*
|
||||
* @author Chris Schaefer
|
||||
*/
|
||||
public class PersonItemProcessorTests {
|
||||
private static final String FIRST_NAME = "jane";
|
||||
private static final String LAST_NAME = "doe";
|
||||
|
||||
@Test
|
||||
public void testPersonProcessing() throws Exception {
|
||||
Person person = new Person(FIRST_NAME, LAST_NAME);
|
||||
|
||||
ItemProcessor<Person, Person> personItemProcessor = new PersonItemProcessor();
|
||||
Person transformedPerson = personItemProcessor.process(person);
|
||||
|
||||
assertNotNull("Received null Person", transformedPerson);
|
||||
assertNotNull("Received null first name", transformedPerson.getFirstName());
|
||||
assertNotNull("Received null last name", transformedPerson.getLastName());
|
||||
assertEquals("Invalid first name processing, should be uppercase",
|
||||
person.getFirstName().toUpperCase(), transformedPerson.getFirstName());
|
||||
assertEquals("Invalid last name processing, should be uppercase",
|
||||
person.getLastName().toUpperCase(), transformedPerson.getLastName());
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
= Spring Cloud Data Flow Samples
|
||||
Sabby Anandan; David Turanski; Glenn Renfro; Eric Bottard; Mark Pollack;
|
||||
Sabby Anandan; David Turanski; Glenn Renfro; Eric Bottard; Mark Pollack; Chris Schaefer
|
||||
:doctype: book
|
||||
:toc:
|
||||
:toclevels: 4
|
||||
|
||||
@@ -12,6 +12,7 @@ include::streaming/custom-apps/celsius-converter-processor/main.adoc[]
|
||||
|
||||
== Task / Batch
|
||||
include::tasks/simple-batch-job/main.adoc[]
|
||||
include::tasks/file-ingest/main.adoc[]
|
||||
|
||||
== Analytics
|
||||
include::analytics/twitter-analytics/main.adoc[]
|
||||
|
||||
121
src/main/asciidoc/tasks/file-ingest/main.adoc
Normal file
121
src/main/asciidoc/tasks/file-ingest/main.adoc
Normal file
@@ -0,0 +1,121 @@
|
||||
|
||||
=== Batch File Ingest
|
||||
|
||||
In this demonstration, you will learn how to create a data processing application using http://projects.spring.io/spring-batch/[Spring Batch] which will then be run within http://cloud.spring.io/spring-cloud-dataflow/[Spring Cloud Data Flow].
|
||||
|
||||
==== Prerequisites
|
||||
|
||||
* A Running Data Flow Server
|
||||
include::{docs_dir}/local-server.adoc[]
|
||||
|
||||
* A Running Data Flow Shell
|
||||
include::{docs_dir}/shell.adoc[]
|
||||
|
||||
==== Batch File Ingest Demo Overview
|
||||
|
||||
The source for the demo project is located in the `batch/file-ingest` directory at the top-level of this repository. The sample is a Spring Boot application that demonstrates how to read data from a flat file, perform processing on the records, and store the transformed data into a database using Spring Batch.
|
||||
|
||||
The key classes for creating the batch job are:
|
||||
|
||||
* `BatchConfiguration.java` - this is where we define our batch job, the step and components that are used read, process, and write our data. In the sample we use a `FlatFileItemReader` which reads a delimited file, a custom `PersonItemProcessor` to transform the data, and a `JdbcBatchItemWriter` to write our data to a database.
|
||||
|
||||
* `Person.java` - the domain object representing the data we are reading and processing in our batch job. The sample data contains records made up of a persons first and last name.
|
||||
|
||||
* `PersonItemProcessor.java` - this class is an `ItemProcessor` implementation which receives records after they have been read and before they are written. This allows us to transform the data between these two steps. In our sample `ItemProcessor` implementation, we simply transform the first and last name of each `Person` to uppercase characters.
|
||||
|
||||
* `Application.java` - the main entry point into the Spring Boot application which is used to launch the batch job
|
||||
|
||||
Resource files are included to set up the database and provide sample data:
|
||||
|
||||
* `schema-all.sql` - this is the database schema that will be created when the application starts up. In this sample, an in-memory database is created on start up and destroyed when the application exits.
|
||||
|
||||
* `data.csv` - sample data file containing person records used in the demo
|
||||
|
||||
|
||||
==== Building and Running the Demo
|
||||
|
||||
. Build the demo JAR
|
||||
+
|
||||
```
|
||||
$ mvn clean package
|
||||
```
|
||||
|
||||
+
|
||||
|
||||
. Register the task
|
||||
+
|
||||
```
|
||||
dataflow:>app register --name fileIngest --type task --uri file:////path/to/target/ingest-X.X.X.jar
|
||||
Successfully registered application 'task:fileIngest'
|
||||
dataflow:>
|
||||
```
|
||||
|
||||
+
|
||||
|
||||
. Create the task
|
||||
+
|
||||
```
|
||||
dataflow:>task create fileIngestTask --definition fileIngest
|
||||
Created new task 'fileIngestTask'
|
||||
dataflow:>
|
||||
```
|
||||
|
||||
+
|
||||
|
||||
. Launch the task
|
||||
+
|
||||
```
|
||||
dataflow:>task launch fileIngestTask --arguments "filePath=classpath:data.csv"
|
||||
Launched task 'fileIngestTask'
|
||||
dataflow:>
|
||||
```
|
||||
|
||||
+
|
||||
|
||||
. Inspect logs
|
||||
+
|
||||
The log file path for the launched task can be found in the local server output, for example:
|
||||
+
|
||||
|
||||
[source,console,options=nowrap]
|
||||
----
|
||||
2017-10-27 14:58:18.112 INFO 19485 --- [nio-9393-exec-6] o.s.c.d.spi.local.LocalTaskLauncher : launching task fileIngestTask-8932f73d-f17a-4bba-b44d-3fd9df042ac0
|
||||
Logs will be in /var/folders/6x/tgtx9xbn0x16xq2sx1j2rld80000gn/T/spring-cloud-dataflow-983191515779755562/fileIngestTask-1509130698071/fileIngestTask-8932f73d-f17a-4bba-b44d-3fd9df042ac0
|
||||
----
|
||||
|
||||
. Verify Task execution details
|
||||
|
||||
+
|
||||
|
||||
[source,console,options=nowrap]
|
||||
----
|
||||
dataflow:>task execution list
|
||||
╔══════════════╤══╤════════════════════════════╤════════════════════════════╤═════════╗
|
||||
║ Task Name │ID│ Start Time │ End Time │Exit Code║
|
||||
╠══════════════╪══╪════════════════════════════╪════════════════════════════╪═════════╣
|
||||
║fileIngestTask│1 │Fri Oct 27 14:58:20 EDT 2017│Fri Oct 27 14:58:20 EDT 2017│0 ║
|
||||
╚══════════════╧══╧════════════════════════════╧════════════════════════════╧═════════╝
|
||||
----
|
||||
|
||||
. Verify Job execution details
|
||||
|
||||
+
|
||||
|
||||
[source,console,options=nowrap]
|
||||
----
|
||||
dataflow:>job execution list
|
||||
╔═══╤═══════╤═════════╤════════════════════════════╤═════════════════════╤══════════════════╗
|
||||
║ID │Task ID│Job Name │ Start Time │Step Execution Count │Definition Status ║
|
||||
╠═══╪═══════╪═════════╪════════════════════════════╪═════════════════════╪══════════════════╣
|
||||
║1 │1 │ingestJob│Fri Oct 27 14:58:20 EDT 2017│1 │Created ║
|
||||
╚═══╧═══════╧═════════╧════════════════════════════╧═════════════════════╧══════════════════╝
|
||||
----
|
||||
|
||||
|
||||
==== Summary
|
||||
|
||||
In this sample, you have learned:
|
||||
|
||||
* How to create a data processing batch job application
|
||||
* How to register and orchestrate Spring Batch jobs in Spring Cloud Data Flow
|
||||
* How to verify status via logs and shell commands
|
||||
Reference in New Issue
Block a user