Add metrics sample
Issue BATCH-2774
This commit is contained in:
Mahmoud Ben Hassine
committed by
Michael Minella
Michael Minella
parent
4de1a9056e
commit
96ec0d0d27
@@ -99,6 +99,7 @@ allprojects {
|
||||
jaxbApiVersion = '2.3.1'
|
||||
jaxbImplVersion = '2.3.0.1'
|
||||
micrometerVersion = '1.1.4'
|
||||
prometheusPushgatewayVersion = '0.6.0'
|
||||
docResourcesVersion = '0.1.1.RELEASE'
|
||||
}
|
||||
|
||||
@@ -596,6 +597,8 @@ project('spring-batch-samples') {
|
||||
compile "javax.mail:javax.mail-api:$javaMailVersion"
|
||||
compile "org.apache.activemq:activemq-client:$activemqVersion"
|
||||
compile "org.apache.activemq:activemq-broker:$activemqVersion"
|
||||
compile "io.micrometer:micrometer-registry-prometheus:$micrometerVersion"
|
||||
compile "io.prometheus:simpleclient_pushgateway:$prometheusPushgatewayVersion"
|
||||
|
||||
testCompile "org.xmlunit:xmlunit-core:$xmlunitVersion"
|
||||
testCompile "org.xmlunit:xmlunit-matchers:$xmlunitVersion"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
Lucas Ward, Dave Syer, Thomas Risberg, Robert Kasanicky, Dan Garrette, Wayne Lund,
|
||||
Michael Minella, Chris Schaefer, Gunnar Hillert, Glenn Renfro, Jay Bryant, Mahmoud Ben Hassine
|
||||
|
||||
Copyright © 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Pivotal, Inc. All Rights
|
||||
Copyright © 2009 - 2019 Pivotal, Inc. All Rights
|
||||
Reserved.
|
||||
|
||||
Copies of this document may be made for your own use and for
|
||||
|
||||
@@ -10,7 +10,7 @@ The reference documentation is divided into several sections:
|
||||
[horizontal]
|
||||
<<spring-batch-intro.adoc#spring-batch-intro,Spring Batch Introduction>> :: Background, usage
|
||||
scenarios and general guidelines.
|
||||
<<whatsnew.adoc#whatsNew,What's new in Spring Batch 4.1>> :: New features introduced in version 4.1.
|
||||
<<whatsnew.adoc#whatsNew,What's new in Spring Batch 4.2>> :: New features introduced in version 4.2.
|
||||
<<domain.adoc#domainLanguageOfBatch,The Domain Language of Batch>> :: Core concepts and abstractions
|
||||
of the Batch domain language.
|
||||
<<job.adoc#configureJob,Configuring and Running a Job>> :: Job configuration, execution and
|
||||
|
||||
@@ -17,7 +17,7 @@ which metrics are provided out-of-the-box and how to contribute custom metrics.
|
||||
Metrics collection does not require any specific configuration. All metrics provided
|
||||
by the framework are registered in
|
||||
link:$$https://micrometer.io/docs/concepts#_global_registry$$[Micrometer's global registry]
|
||||
under the `spring.batch.` prefix. The following table explains all metrics in details:
|
||||
under the `spring.batch` prefix. The following table explains all the metrics in details:
|
||||
|
||||
|===============
|
||||
|__Metric Name__|__Type__|__Description__
|
||||
|
||||
@@ -4,323 +4,28 @@
|
||||
|
||||
[[whatsNew]]
|
||||
|
||||
== What's New in Spring Batch 4.1
|
||||
== What's New in Spring Batch 4.2
|
||||
|
||||
The Spring Batch 4.1 release adds the following features:
|
||||
Spring Batch 4.2 adds the following features:
|
||||
|
||||
* A new `@SpringBatchTest` annotation to simplify testing batch components
|
||||
* A new `@EnableBatchIntegration` annotation to simplify remote chunking and partitioning configuration
|
||||
* A new `JsonItemReader` and `JsonFileItemWriter` to support the JSON format
|
||||
* Add support for validating items with the Bean Validation API
|
||||
* Add support for JSR-305 annotations
|
||||
* Enhancements to the `FlatFileItemWriterBuilder` API
|
||||
* Support for batch metrics with https://micrometer.io[Micrometer]
|
||||
* Improved documentation
|
||||
|
||||
[[whatsNewTesting]]
|
||||
=== `@SpringBatchTest` Annotation
|
||||
[[whatsNewMetrics]]
|
||||
=== Batch metrics with Micrometer
|
||||
|
||||
Spring Batch provides some nice utility classes (such as the `JobLauncherTestUtils` and
|
||||
`JobRepositoryTestUtils`) and test execution listeners (`StepScopeTestExecutionListener`
|
||||
and `JobScopeTestExecutionListener`) to test batch components. However, in order
|
||||
to use these utilities, you must configure them explicitly. This release introduces
|
||||
a new annotation named `@SpringBatchTest` that automatically adds utility beans and
|
||||
listeners to the test context and makes them available for autowiring,
|
||||
as the following example shows:
|
||||
This release introduces a new feature that lets you monitor your batch jobs
|
||||
by using Micrometer. By default, Spring Batch collects metrics (such as job duration,
|
||||
step duration, item read and write throughput, and others) and registers them in Micrometer's
|
||||
global metrics registry under the `spring.batch` prefix.
|
||||
These metrics can be sent to any https://micrometer.io/docs/concepts#_supported_monitoring_systems[monitoring system]
|
||||
supported by Micrometer.
|
||||
|
||||
[source, java]
|
||||
----
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBatchTest
|
||||
@ContextConfiguration(classes = {JobConfiguration.class})
|
||||
public class JobTest {
|
||||
For more details about this feature, please refer to the
|
||||
<<monitoring-and-metrics.adoc#monitoring-and-metrics,Monitoring and metrics>> chapter.
|
||||
|
||||
@Autowired
|
||||
private JobLauncherTestUtils jobLauncherTestUtils;
|
||||
[[whatsNewDocs]]
|
||||
=== Documentation updates
|
||||
|
||||
@Autowired
|
||||
private JobRepositoryTestUtils jobRepositoryTestUtils;
|
||||
|
||||
|
||||
@Before
|
||||
public void clearMetadata() {
|
||||
jobRepositoryTestUtils.removeJobExecutions();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJob() throws Exception {
|
||||
// given
|
||||
JobParameters jobParameters =
|
||||
jobLauncherTestUtils.getUniqueJobParameters();
|
||||
|
||||
// when
|
||||
JobExecution jobExecution =
|
||||
jobLauncherTestUtils.launchJob(jobParameters);
|
||||
|
||||
// then
|
||||
Assert.assertEquals(ExitStatus.COMPLETED,
|
||||
jobExecution.getExitStatus());
|
||||
}
|
||||
|
||||
}
|
||||
----
|
||||
|
||||
For more details about this new annotation, see the
|
||||
<<testing.adoc#creatingUnitTestClass,Unit Testing>> chapter.
|
||||
|
||||
[[whatsNewIntegration]]
|
||||
=== `@EnableBatchIntegration` Annotation
|
||||
|
||||
Setting up a remote chunking job requires the definition of a number of beans:
|
||||
|
||||
* A connection factory to acquire connections from the messaging middleware (JMS, AMQP, and others)
|
||||
* A `MessagingTemplate` to send requests from the master to the workers and back again
|
||||
* An input channel and an output channel for Spring Integration to get messages from the messaging middleware
|
||||
* A special item writer (`ChunkMessageChannelItemWriter`) on the master side that knows how to send chunks of data to workers for processing and writing
|
||||
* A message listener (`ChunkProcessorChunkHandler`) on the worker side to receive data from the master
|
||||
|
||||
This can be a bit daunting at first glance. This release introduces a new annotation
|
||||
named `@EnableBatchIntegration` as well as new APIs (`RemoteChunkingMasterStepBuilder`
|
||||
and `RemoteChunkingWorkerBuilder`) to simplify the configuration. The following
|
||||
example shows how to use the new annotation and APIs:
|
||||
|
||||
[source, java]
|
||||
----
|
||||
@Configuration
|
||||
@EnableBatchProcessing
|
||||
@EnableBatchIntegration
|
||||
public class RemoteChunkingAppConfig {
|
||||
|
||||
@Autowired
|
||||
private RemoteChunkingMasterStepBuilderFactory masterStepBuilderFactory;
|
||||
|
||||
@Autowired
|
||||
private RemoteChunkingWorkerBuilder workerBuilder;
|
||||
|
||||
@Bean
|
||||
public TaskletStep masterStep() {
|
||||
return this.masterStepBuilderFactory
|
||||
.get("masterStep")
|
||||
.chunk(100)
|
||||
.reader(itemReader())
|
||||
.outputChannel(outgoingRequestsToWorkers())
|
||||
.inputChannel(incomingRepliesFromWorkers())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public IntegrationFlow worker() {
|
||||
return this.workerBuilder
|
||||
.itemProcessor(itemProcessor())
|
||||
.itemWriter(itemWriter())
|
||||
.inputChannel(incomingRequestsFromMaster())
|
||||
.outputChannel(outgoingRepliesToMaster())
|
||||
.build();
|
||||
}
|
||||
|
||||
// Middleware beans setup omitted
|
||||
}
|
||||
----
|
||||
|
||||
This new annotation and builders take care of the heavy lifting of configuring
|
||||
infrastructure beans. You can now easily configure a master step as well as
|
||||
a Spring Integration flow on the worker side. You can find a remote chunking sample
|
||||
that uses these new APIs in the
|
||||
link:$$https://github.com/spring-projects/spring-batch/tree/master/spring-batch-samples#remote-chunking-sample$$[samples module]
|
||||
as well as more details in the <<spring-batch-integration.adoc#remote-chunking,Spring Batch Integration>> chapter.
|
||||
|
||||
Just like the remote chunking configuration simplification, this version also
|
||||
introduces new APIs to simplify a remote partitioning setup:
|
||||
`RemotePartitioningMasterStepBuilder` and `RemotePartitioningWorkerStepBuilder`.
|
||||
Those can be autowired in your configuration class if the
|
||||
`@EnableBatchIntegration` is present as shown in the following example:
|
||||
|
||||
[source, java]
|
||||
----
|
||||
@Configuration
|
||||
@EnableBatchProcessing
|
||||
@EnableBatchIntegration
|
||||
public class RemotePartitioningAppConfig {
|
||||
|
||||
@Autowired
|
||||
private RemotePartitioningMasterStepBuilderFactory masterStepBuilderFactory;
|
||||
|
||||
@Autowired
|
||||
private RemotePartitioningWorkerStepBuilderFactory workerStepBuilderFactory;
|
||||
|
||||
@Bean
|
||||
public Step masterStep() {
|
||||
return this.masterStepBuilderFactory
|
||||
.get("masterStep")
|
||||
.partitioner("workerStep", partitioner())
|
||||
.gridSize(10)
|
||||
.outputChannel(outgoingRequestsToWorkers())
|
||||
.inputChannel(incomingRepliesFromWorkers())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Step workerStep() {
|
||||
return this.workerStepBuilderFactory
|
||||
.get("workerStep")
|
||||
.inputChannel(incomingRequestsFromMaster())
|
||||
.outputChannel(outgoingRepliesToMaster())
|
||||
.chunk(100)
|
||||
.reader(itemReader())
|
||||
.processor(itemProcessor())
|
||||
.writer(itemWriter())
|
||||
.build();
|
||||
}
|
||||
|
||||
// Middleware beans setup omitted
|
||||
}
|
||||
----
|
||||
|
||||
You can find more details about these new APIs in the <<spring-batch-integration.adoc#remote-partitioning,Spring Batch Integration>> chapter.
|
||||
|
||||
[[whatsNewJson]]
|
||||
=== JSON support
|
||||
|
||||
Spring Batch 4.1 adds support for the JSON format. This release introduces a new
|
||||
item reader that can read a JSON resource in the following format:
|
||||
|
||||
[source, json]
|
||||
----
|
||||
[
|
||||
{
|
||||
"isin": "123",
|
||||
"quantity": 1,
|
||||
"price": 1.2,
|
||||
"customer": "foo"
|
||||
},
|
||||
{
|
||||
"isin": "456",
|
||||
"quantity": 2,
|
||||
"price": 1.4,
|
||||
"customer": "bar"
|
||||
}
|
||||
]
|
||||
----
|
||||
|
||||
Similar to the `StaxEventItemReader` for XML, the new `JsonItemReader` uses streaming
|
||||
APIs to read JSON objects in chunks. Spring Batch supports two libraries:
|
||||
|
||||
* link:$$https://github.com/FasterXML/jackson$$[Jackson]
|
||||
* link:$$https://github.com/google/gson$$[Gson]
|
||||
|
||||
To add other libraries, you can implement the `JsonObjectReader` interface.
|
||||
|
||||
Writing JSON data is also supported through the `JsonFileItemWriter`.
|
||||
For more details about JSON support, see the
|
||||
<<readersAndWriters.adoc#jsonReadingWriting,ItemReaders and ItemWriters>> chapter.
|
||||
|
||||
[[whatsNewBeanValidationApi]]
|
||||
=== Bean Validation API support
|
||||
|
||||
This release brings a new `ValidatingItemProcessor` implementation called
|
||||
`BeanValidatingItemProcessor` which allows you to validate items annotated with
|
||||
the Bean Validation API (JSR-303) annotations. For example, given the following
|
||||
type `Person`:
|
||||
|
||||
[source, java]
|
||||
----
|
||||
class Person {
|
||||
|
||||
@NotEmpty
|
||||
private String name;
|
||||
|
||||
public Person(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
}
|
||||
----
|
||||
|
||||
you can validate items by declaring a `BeanValidatingItemProcessor` bean in your
|
||||
application context and register it as a processor in your chunk-oriented step:
|
||||
|
||||
[source, java]
|
||||
----
|
||||
@Bean
|
||||
public BeanValidatingItemProcessor<Person> beanValidatingItemProcessor() throws Exception {
|
||||
BeanValidatingItemProcessor<Person> beanValidatingItemProcessor = new BeanValidatingItemProcessor<>();
|
||||
beanValidatingItemProcessor.setFilter(true);
|
||||
|
||||
return beanValidatingItemProcessor;
|
||||
}
|
||||
----
|
||||
|
||||
[[whatsNewJSR305Api]]
|
||||
=== JSR-305 support
|
||||
|
||||
This release adds support for JSR-305 annotations. It leverages Spring Framework’s
|
||||
link:$$https://docs.spring.io/spring/docs/current/spring-framework-reference/core.html#null-safety$$[Null-safety]
|
||||
annotations and adds them on all public APIs of Spring Batch.
|
||||
|
||||
These annotations will not only enforce null-safety when using Spring Batch APIs,
|
||||
but also can be used by IDEs to provide useful information related to nullability.
|
||||
For example, if a user wants to implement the `ItemReader` interface, any IDE
|
||||
supporting JSR-305 annotations will generate something like:
|
||||
|
||||
[source, java]
|
||||
----
|
||||
public class MyItemReader implements ItemReader<String> {
|
||||
|
||||
@Nullable
|
||||
public String read() throws Exception {
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
----
|
||||
|
||||
The `@Nullable` annotation present on the `read` method makes it clear that the
|
||||
contract of this method says it may return `null`. This enforces what is said in its
|
||||
Javadoc, that the `read` method should return `null` when the data source is exhausted.
|
||||
|
||||
[[whatsNewFlatFileItemWriterBuilder]]
|
||||
=== `FlatFileItemWriterBuilder` enhancements
|
||||
|
||||
Another small feature added in this release is a simplification of the configuration
|
||||
for the writing of a flat file. Specifically, these updates simplify the configuration
|
||||
of both a delimited and fixed width file. Below is an example of before and after the change.
|
||||
|
||||
[source, java]
|
||||
----
|
||||
// Before
|
||||
@Bean
|
||||
public FlatFileItemWriter<Item> itemWriter(Resource resource) {
|
||||
BeanWrapperFieldExtractor<Item> fieldExtractor =
|
||||
new BeanWrapperFieldExtractor<Item>();
|
||||
fieldExtractor.setNames(new String[] {"field1", "field2", "field3"});
|
||||
fieldExtractor.afterPropertiesSet();
|
||||
|
||||
DelimitedLineAggregator aggregator = new DelimitedLineAggregator();
|
||||
aggregator.setFieldExtractor(fieldExtractor);
|
||||
aggregator.setDelimiter(";");
|
||||
|
||||
return new FlatFileItemWriterBuilder<Item>()
|
||||
.name("itemWriter")
|
||||
.resource(resource)
|
||||
.lineAggregator(aggregator)
|
||||
.build();
|
||||
}
|
||||
|
||||
// After
|
||||
@Bean
|
||||
public FlatFileItemWriter<Item> itemWriter(Resource resource) {
|
||||
return new FlatFileItemWriterBuilder<Item>()
|
||||
.name("itemWriter")
|
||||
.resource(resource)
|
||||
.delimited()
|
||||
.delimiter(";")
|
||||
.names(new String[] {"field1", "field2", "field3"})
|
||||
.build();
|
||||
}
|
||||
----
|
||||
The reference documentation has been updated to match the same style as other
|
||||
Spring projects.
|
||||
|
||||
@@ -888,3 +888,34 @@ file to another. It uses XStream for the object XML conversion,
|
||||
because this is simple to configure for basic use cases like this
|
||||
one. See
|
||||
[Spring OXM documentation](https://docs.spring.io/spring/docs/current/spring-framework-reference/data-access.html#oxm) for details of other options.
|
||||
|
||||
### Batch metrics with Micrometer
|
||||
|
||||
This sample shows how to use [Micrometer](https://micrometer.io) to collect batch metrics in Spring Batch.
|
||||
It uses [Prometheus](https://prometheus.io) as the metrics back end and [Grafana](https://grafana.com) as the front end.
|
||||
The sample consists of two jobs:
|
||||
|
||||
* `job1` : Composed of two tasklets that print `hello` and `world`
|
||||
* `job2` : Composed of single chunk-oriented step that reads and writes a random number of items
|
||||
|
||||
These two jobs are run repeatedly at regular intervals and might fail randomly for demonstration purposes.
|
||||
|
||||
This sample requires [docker compose](https://docs.docker.com/compose/) to start the monitoring stack.
|
||||
To run the sample, please follow these steps:
|
||||
|
||||
```
|
||||
$>cd spring-batch-samples/src/grafana
|
||||
$>docker-compose up -d
|
||||
```
|
||||
|
||||
This should start the required monitoring stack:
|
||||
|
||||
* Prometheus server on port `9090`
|
||||
* Prometheus push gateway on port `9091`
|
||||
* Grafana on port `3000`
|
||||
|
||||
Once started, you need to [configure Prometheus as data source in Grafana](https://grafana.com/docs/features/datasources/prometheus/)
|
||||
and import the ready-to-use dashboard in `spring-batch-samples/src/grafana/spring-batch-dashboard.json`.
|
||||
|
||||
Finally, run the `org.springframework.batch.sample.metrics.BatchMetricsApplication`
|
||||
class without any argument to start the sample.
|
||||
|
||||
22
spring-batch-samples/src/grafana/docker-compose.yml
Normal file
22
spring-batch-samples/src/grafana/docker-compose.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
version: '3.3'
|
||||
services:
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.7.2
|
||||
container_name: 'prometheus'
|
||||
ports:
|
||||
- '9090:9090'
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
|
||||
pushgateway:
|
||||
image: prom/pushgateway:v0.6.0
|
||||
container_name: 'pushgateway'
|
||||
ports:
|
||||
- '9091:9091'
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:6.0.2
|
||||
container_name: 'grafana'
|
||||
ports:
|
||||
- '3000:3000'
|
||||
@@ -0,0 +1,9 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'springbatch'
|
||||
honor_labels: true
|
||||
static_configs:
|
||||
- targets: ['pushgateway:9091']
|
||||
568
spring-batch-samples/src/grafana/spring-batch-dashboard.json
Normal file
568
spring-batch-samples/src/grafana/spring-batch-dashboard.json
Normal file
@@ -0,0 +1,568 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"limit": 100,
|
||||
"name": "Annotations & Alerts",
|
||||
"showIn": 0,
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Dashboard for Spring Batch applications instrumented with Micrometer",
|
||||
"editable": true,
|
||||
"gnetId": 4701,
|
||||
"graphTooltip": 1,
|
||||
"id": 1,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 132,
|
||||
"panels": [],
|
||||
"repeat": null,
|
||||
"title": "Spring Batch Metrics",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
"grid": {
|
||||
"leftLogBase": 1,
|
||||
"leftMax": null,
|
||||
"leftMin": null,
|
||||
"rightLogBase": 1,
|
||||
"rightMax": null,
|
||||
"rightMin": null
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 37,
|
||||
"legend": {
|
||||
"alignAsTable": false,
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"hideEmpty": false,
|
||||
"hideZero": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"rightSide": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"paceLength": 10,
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "spring_batch_job_seconds_max",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "{{name}} {{status}}",
|
||||
"metric": "",
|
||||
"refId": "A",
|
||||
"step": 1200
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Job duration",
|
||||
"tooltip": {
|
||||
"msResolution": false,
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "cumulative"
|
||||
},
|
||||
"type": "graph",
|
||||
"x-axis": true,
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"y-axis": true,
|
||||
"y_formats": [
|
||||
"short",
|
||||
"short"
|
||||
],
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "s",
|
||||
"label": "",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": "",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
"grid": {
|
||||
"leftLogBase": 1,
|
||||
"leftMax": null,
|
||||
"leftMin": null,
|
||||
"rightLogBase": 1,
|
||||
"rightMax": null,
|
||||
"rightMin": null
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 1
|
||||
},
|
||||
"id": 38,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"paceLength": 10,
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(spring_batch_job_seconds_count[1m]) * 60",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{name}} {{status}}",
|
||||
"metric": "",
|
||||
"refId": "A",
|
||||
"step": 1200
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Job execution rate",
|
||||
"tooltip": {
|
||||
"msResolution": false,
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "cumulative"
|
||||
},
|
||||
"type": "graph",
|
||||
"x-axis": true,
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"y-axis": true,
|
||||
"y_formats": [
|
||||
"ops",
|
||||
"short"
|
||||
],
|
||||
"yaxes": [
|
||||
{
|
||||
"decimals": null,
|
||||
"format": "opm",
|
||||
"label": "",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 1
|
||||
},
|
||||
"id": 138,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"paceLength": 10,
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(sum(rate(spring_batch_job_seconds_count{status=\"FAILED\"}[5m])) by (name, status)) / (sum(rate(spring_batch_job_seconds_count[5m])) by (name, status))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "{{name}} {{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Job failure rate",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "percentunit",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 141,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"paceLength": 10,
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [
|
||||
{
|
||||
"alias": "write success",
|
||||
"yaxis": 1
|
||||
}
|
||||
],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(spring_batch_item_read_seconds_count[1m])",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "read {{status}}",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Item read throughput",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "ops",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "ops",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"id": 140,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"paceLength": 10,
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [
|
||||
{
|
||||
"alias": "write success",
|
||||
"yaxis": 1
|
||||
}
|
||||
],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(spring_batch_chunk_write_seconds_count[1m])",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "write {{status}}",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Item write throughput",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "ops",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "ops",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 18,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-5m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"now": true,
|
||||
"refresh_intervals": [
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
],
|
||||
"time_options": [
|
||||
"5m",
|
||||
"15m",
|
||||
"1h",
|
||||
"6h",
|
||||
"12h",
|
||||
"24h",
|
||||
"2d",
|
||||
"7d",
|
||||
"30d"
|
||||
]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "Spring Batch Prometheus",
|
||||
"uid": "qRLUmOCmk",
|
||||
"version": 6
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package org.springframework.batch.sample.metrics;
|
||||
|
||||
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.PropertySource;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
|
||||
|
||||
@EnableScheduling
|
||||
@EnableBatchProcessing
|
||||
@Import({Job1Configuration.class, Job2Configuration.class, JobScheduler.class, PrometheusConfiguration.class})
|
||||
@PropertySource("metrics-sample.properties")
|
||||
public class BatchMetricsApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
AnnotationConfigApplicationContext applicationContext = new AnnotationConfigApplicationContext(BatchMetricsApplication.class);
|
||||
applicationContext.start();
|
||||
}
|
||||
|
||||
@Bean(destroyMethod = "shutdown")
|
||||
public ThreadPoolTaskScheduler taskScheduler(@Value("${thread.pool.size}") int threadPoolSize) {
|
||||
ThreadPoolTaskScheduler threadPoolTaskScheduler = new ThreadPoolTaskScheduler();
|
||||
threadPoolTaskScheduler.setPoolSize(threadPoolSize);
|
||||
return threadPoolTaskScheduler;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package org.springframework.batch.sample.metrics;
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
import org.springframework.batch.core.Job;
|
||||
import org.springframework.batch.core.Step;
|
||||
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
|
||||
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
|
||||
import org.springframework.batch.repeat.RepeatStatus;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class Job1Configuration {
|
||||
|
||||
private Random random;
|
||||
private JobBuilderFactory jobs;
|
||||
private StepBuilderFactory steps;
|
||||
|
||||
public Job1Configuration(JobBuilderFactory jobs, StepBuilderFactory steps) {
|
||||
this.jobs = jobs;
|
||||
this.steps = steps;
|
||||
this.random = new Random();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Job job1() {
|
||||
return jobs.get("job1")
|
||||
.start(step1())
|
||||
.next(step2())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Step step1() {
|
||||
return steps.get("step1")
|
||||
.tasklet((contribution, chunkContext) -> {
|
||||
System.out.println("hello");
|
||||
// simulate processing time
|
||||
Thread.sleep(random.nextInt(3000));
|
||||
return RepeatStatus.FINISHED;
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Step step2() {
|
||||
return steps.get("step2")
|
||||
.tasklet((contribution, chunkContext) -> {
|
||||
System.out.println("world");
|
||||
// simulate step failure
|
||||
int nextInt = random.nextInt(3000);
|
||||
Thread.sleep(nextInt);
|
||||
if (nextInt % 5 == 0) {
|
||||
throw new Exception("Boom!");
|
||||
}
|
||||
return RepeatStatus.FINISHED;
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package org.springframework.batch.sample.metrics;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import org.springframework.batch.core.Job;
|
||||
import org.springframework.batch.core.Step;
|
||||
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
|
||||
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
|
||||
import org.springframework.batch.core.configuration.annotation.StepScope;
|
||||
import org.springframework.batch.item.ItemWriter;
|
||||
import org.springframework.batch.item.support.ListItemReader;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class Job2Configuration {
|
||||
|
||||
private Random random;
|
||||
private JobBuilderFactory jobs;
|
||||
private StepBuilderFactory steps;
|
||||
|
||||
public Job2Configuration(JobBuilderFactory jobs, StepBuilderFactory steps) {
|
||||
this.jobs = jobs;
|
||||
this.steps = steps;
|
||||
this.random = new Random();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Job job2() {
|
||||
return jobs.get("job2")
|
||||
.start(step())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Step step() {
|
||||
return steps.get("step1")
|
||||
.<Integer, Integer>chunk(3)
|
||||
.reader(itemReader())
|
||||
.writer(itemWriter())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@StepScope
|
||||
public ListItemReader<Integer> itemReader() {
|
||||
List<Integer> items = new LinkedList<>();
|
||||
// read a random number of items in each run
|
||||
for (int i = 0; i < random.nextInt(100); i++) {
|
||||
items.add(i);
|
||||
}
|
||||
return new ListItemReader<>(items);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ItemWriter<Integer> itemWriter() {
|
||||
return items -> {
|
||||
for (Integer item : items) {
|
||||
int nextInt = random.nextInt(1000);
|
||||
Thread.sleep(nextInt);
|
||||
// simulate write failure
|
||||
if (nextInt % 57 == 0) {
|
||||
throw new Exception("Boom!");
|
||||
}
|
||||
System.out.println("item = " + item);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package org.springframework.batch.sample.metrics;
|
||||
|
||||
import org.springframework.batch.core.Job;
|
||||
import org.springframework.batch.core.JobParameters;
|
||||
import org.springframework.batch.core.JobParametersBuilder;
|
||||
import org.springframework.batch.core.launch.JobLauncher;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class JobScheduler {
|
||||
|
||||
private final Job job1;
|
||||
private final Job job2;
|
||||
private final JobLauncher jobLauncher;
|
||||
|
||||
@Autowired
|
||||
public JobScheduler(Job job1, Job job2, JobLauncher jobLauncher) {
|
||||
this.job1 = job1;
|
||||
this.job2 = job2;
|
||||
this.jobLauncher = jobLauncher;
|
||||
}
|
||||
|
||||
@Scheduled(cron="*/10 * * * * *")
|
||||
public void launchJob1() throws Exception {
|
||||
JobParameters jobParameters = new JobParametersBuilder()
|
||||
.addLong("time", System.currentTimeMillis())
|
||||
.toJobParameters();
|
||||
|
||||
jobLauncher.run(job1, jobParameters);
|
||||
}
|
||||
|
||||
@Scheduled(cron="*/15 * * * * *")
|
||||
public void launchJob2() throws Exception {
|
||||
JobParameters jobParameters = new JobParametersBuilder()
|
||||
.addLong("time", System.currentTimeMillis())
|
||||
.toJobParameters();
|
||||
|
||||
jobLauncher.run(job2, jobParameters);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package org.springframework.batch.sample.metrics;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
import io.micrometer.core.instrument.Metrics;
|
||||
import io.micrometer.prometheus.PrometheusConfig;
|
||||
import io.micrometer.prometheus.PrometheusMeterRegistry;
|
||||
import io.prometheus.client.CollectorRegistry;
|
||||
import io.prometheus.client.exporter.PushGateway;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
|
||||
@Configuration
|
||||
public class PrometheusConfiguration {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusConfiguration.class);
|
||||
|
||||
@Value("${prometheus.job.name}")
|
||||
private String prometheusJobName;
|
||||
|
||||
@Value("${prometheus.grouping.key}")
|
||||
private String prometheusGroupingKey;
|
||||
|
||||
@Value("${prometheus.pushgateway.url}")
|
||||
private String prometheusPushGatewayUrl;
|
||||
|
||||
private Map<String, String> groupingKey = new HashMap<>();
|
||||
private PushGateway pushGateway;
|
||||
private CollectorRegistry collectorRegistry;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
pushGateway = new PushGateway(prometheusPushGatewayUrl);
|
||||
groupingKey.put(prometheusGroupingKey, prometheusJobName);
|
||||
PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
|
||||
collectorRegistry = prometheusMeterRegistry.getPrometheusRegistry();
|
||||
Metrics.globalRegistry.add(prometheusMeterRegistry);
|
||||
}
|
||||
|
||||
@Scheduled(fixedRateString = "${prometheus.push.rate}")
|
||||
public void pushMetrics() {
|
||||
try {
|
||||
pushGateway.pushAdd(collectorRegistry, prometheusJobName, groupingKey);
|
||||
} catch (Throwable ex) {
|
||||
LOGGER.error("Unable to push metrics to Prometheus Push Gateway", ex);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
thread.pool.size=3
|
||||
prometheus.push.rate=5000
|
||||
prometheus.job.name=springbatch
|
||||
prometheus.grouping.key=appname
|
||||
prometheus.pushgateway.url=0.0.0.0:9091
|
||||
Reference in New Issue
Block a user