diff --git a/build.gradle b/build.gradle index cd34a0305..f3f50b6e4 100644 --- a/build.gradle +++ b/build.gradle @@ -99,6 +99,7 @@ allprojects { jaxbApiVersion = '2.3.1' jaxbImplVersion = '2.3.0.1' micrometerVersion = '1.1.4' + prometheusPushgatewayVersion = '0.6.0' docResourcesVersion = '0.1.1.RELEASE' } @@ -596,6 +597,8 @@ project('spring-batch-samples') { compile "javax.mail:javax.mail-api:$javaMailVersion" compile "org.apache.activemq:activemq-client:$activemqVersion" compile "org.apache.activemq:activemq-broker:$activemqVersion" + compile "io.micrometer:micrometer-registry-prometheus:$micrometerVersion" + compile "io.prometheus:simpleclient_pushgateway:$prometheusPushgatewayVersion" testCompile "org.xmlunit:xmlunit-core:$xmlunitVersion" testCompile "org.xmlunit:xmlunit-matchers:$xmlunitVersion" diff --git a/spring-batch-docs/asciidoc/footer/index-footer.adoc b/spring-batch-docs/asciidoc/footer/index-footer.adoc index 496e8cdf7..c7bed9e10 100644 --- a/spring-batch-docs/asciidoc/footer/index-footer.adoc +++ b/spring-batch-docs/asciidoc/footer/index-footer.adoc @@ -2,7 +2,7 @@ Lucas Ward, Dave Syer, Thomas Risberg, Robert Kasanicky, Dan Garrette, Wayne Lund, Michael Minella, Chris Schaefer, Gunnar Hillert, Glenn Renfro, Jay Bryant, Mahmoud Ben Hassine -Copyright © 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Pivotal, Inc. All Rights +Copyright © 2009 - 2019 Pivotal, Inc. All Rights Reserved. Copies of this document may be made for your own use and for diff --git a/spring-batch-docs/asciidoc/index.adoc b/spring-batch-docs/asciidoc/index.adoc index 8eb4768f3..3fcde2b23 100644 --- a/spring-batch-docs/asciidoc/index.adoc +++ b/spring-batch-docs/asciidoc/index.adoc @@ -10,7 +10,7 @@ The reference documentation is divided into several sections: [horizontal] <> :: Background, usage scenarios and general guidelines. -<> :: New features introduced in version 4.1. +<> :: New features introduced in version 4.2. <> :: Core concepts and abstractions of the Batch domain language. <> :: Job configuration, execution and diff --git a/spring-batch-docs/asciidoc/monitoring-and-metrics.adoc b/spring-batch-docs/asciidoc/monitoring-and-metrics.adoc index c564f4277..efb015ced 100644 --- a/spring-batch-docs/asciidoc/monitoring-and-metrics.adoc +++ b/spring-batch-docs/asciidoc/monitoring-and-metrics.adoc @@ -17,7 +17,7 @@ which metrics are provided out-of-the-box and how to contribute custom metrics. Metrics collection does not require any specific configuration. All metrics provided by the framework are registered in link:$$https://micrometer.io/docs/concepts#_global_registry$$[Micrometer's global registry] -under the `spring.batch.` prefix. The following table explains all metrics in details: +under the `spring.batch` prefix. The following table explains all the metrics in details: |=============== |__Metric Name__|__Type__|__Description__ diff --git a/spring-batch-docs/asciidoc/whatsnew.adoc b/spring-batch-docs/asciidoc/whatsnew.adoc index b7b75cedb..cc6d8b375 100644 --- a/spring-batch-docs/asciidoc/whatsnew.adoc +++ b/spring-batch-docs/asciidoc/whatsnew.adoc @@ -4,323 +4,28 @@ [[whatsNew]] -== What's New in Spring Batch 4.1 +== What's New in Spring Batch 4.2 -The Spring Batch 4.1 release adds the following features: +Spring Batch 4.2 adds the following features: -* A new `@SpringBatchTest` annotation to simplify testing batch components -* A new `@EnableBatchIntegration` annotation to simplify remote chunking and partitioning configuration -* A new `JsonItemReader` and `JsonFileItemWriter` to support the JSON format -* Add support for validating items with the Bean Validation API -* Add support for JSR-305 annotations -* Enhancements to the `FlatFileItemWriterBuilder` API +* Support for batch metrics with https://micrometer.io[Micrometer] +* Improved documentation -[[whatsNewTesting]] -=== `@SpringBatchTest` Annotation +[[whatsNewMetrics]] +=== Batch metrics with Micrometer -Spring Batch provides some nice utility classes (such as the `JobLauncherTestUtils` and -`JobRepositoryTestUtils`) and test execution listeners (`StepScopeTestExecutionListener` -and `JobScopeTestExecutionListener`) to test batch components. However, in order -to use these utilities, you must configure them explicitly. This release introduces -a new annotation named `@SpringBatchTest` that automatically adds utility beans and -listeners to the test context and makes them available for autowiring, -as the following example shows: +This release introduces a new feature that lets you monitor your batch jobs +by using Micrometer. By default, Spring Batch collects metrics (such as job duration, +step duration, item read and write throughput, and others) and registers them in Micrometer's +global metrics registry under the `spring.batch` prefix. +These metrics can be sent to any https://micrometer.io/docs/concepts#_supported_monitoring_systems[monitoring system] +supported by Micrometer. -[source, java] ----- -@RunWith(SpringRunner.class) -@SpringBatchTest -@ContextConfiguration(classes = {JobConfiguration.class}) -public class JobTest { +For more details about this feature, please refer to the +<> chapter. - @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; +[[whatsNewDocs]] +=== Documentation updates - @Autowired - private JobRepositoryTestUtils jobRepositoryTestUtils; - - - @Before - public void clearMetadata() { - jobRepositoryTestUtils.removeJobExecutions(); - } - - @Test - public void testJob() throws Exception { - // given - JobParameters jobParameters = - jobLauncherTestUtils.getUniqueJobParameters(); - - // when - JobExecution jobExecution = - jobLauncherTestUtils.launchJob(jobParameters); - - // then - Assert.assertEquals(ExitStatus.COMPLETED, - jobExecution.getExitStatus()); - } - -} ----- - -For more details about this new annotation, see the -<> chapter. - -[[whatsNewIntegration]] -=== `@EnableBatchIntegration` Annotation - -Setting up a remote chunking job requires the definition of a number of beans: - -* A connection factory to acquire connections from the messaging middleware (JMS, AMQP, and others) -* A `MessagingTemplate` to send requests from the master to the workers and back again -* An input channel and an output channel for Spring Integration to get messages from the messaging middleware -* A special item writer (`ChunkMessageChannelItemWriter`) on the master side that knows how to send chunks of data to workers for processing and writing -* A message listener (`ChunkProcessorChunkHandler`) on the worker side to receive data from the master - -This can be a bit daunting at first glance. This release introduces a new annotation -named `@EnableBatchIntegration` as well as new APIs (`RemoteChunkingMasterStepBuilder` -and `RemoteChunkingWorkerBuilder`) to simplify the configuration. The following -example shows how to use the new annotation and APIs: - -[source, java] ----- -@Configuration -@EnableBatchProcessing -@EnableBatchIntegration -public class RemoteChunkingAppConfig { - - @Autowired - private RemoteChunkingMasterStepBuilderFactory masterStepBuilderFactory; - - @Autowired - private RemoteChunkingWorkerBuilder workerBuilder; - - @Bean - public TaskletStep masterStep() { - return this.masterStepBuilderFactory - .get("masterStep") - .chunk(100) - .reader(itemReader()) - .outputChannel(outgoingRequestsToWorkers()) - .inputChannel(incomingRepliesFromWorkers()) - .build(); - } - - @Bean - public IntegrationFlow worker() { - return this.workerBuilder - .itemProcessor(itemProcessor()) - .itemWriter(itemWriter()) - .inputChannel(incomingRequestsFromMaster()) - .outputChannel(outgoingRepliesToMaster()) - .build(); - } - - // Middleware beans setup omitted -} ----- - -This new annotation and builders take care of the heavy lifting of configuring -infrastructure beans. You can now easily configure a master step as well as -a Spring Integration flow on the worker side. You can find a remote chunking sample -that uses these new APIs in the -link:$$https://github.com/spring-projects/spring-batch/tree/master/spring-batch-samples#remote-chunking-sample$$[samples module] -as well as more details in the <> chapter. - -Just like the remote chunking configuration simplification, this version also -introduces new APIs to simplify a remote partitioning setup: -`RemotePartitioningMasterStepBuilder` and `RemotePartitioningWorkerStepBuilder`. -Those can be autowired in your configuration class if the -`@EnableBatchIntegration` is present as shown in the following example: - -[source, java] ----- -@Configuration -@EnableBatchProcessing -@EnableBatchIntegration -public class RemotePartitioningAppConfig { - - @Autowired - private RemotePartitioningMasterStepBuilderFactory masterStepBuilderFactory; - - @Autowired - private RemotePartitioningWorkerStepBuilderFactory workerStepBuilderFactory; - - @Bean - public Step masterStep() { - return this.masterStepBuilderFactory - .get("masterStep") - .partitioner("workerStep", partitioner()) - .gridSize(10) - .outputChannel(outgoingRequestsToWorkers()) - .inputChannel(incomingRepliesFromWorkers()) - .build(); - } - - @Bean - public Step workerStep() { - return this.workerStepBuilderFactory - .get("workerStep") - .inputChannel(incomingRequestsFromMaster()) - .outputChannel(outgoingRepliesToMaster()) - .chunk(100) - .reader(itemReader()) - .processor(itemProcessor()) - .writer(itemWriter()) - .build(); - } - - // Middleware beans setup omitted -} ----- - -You can find more details about these new APIs in the <> chapter. - -[[whatsNewJson]] -=== JSON support - -Spring Batch 4.1 adds support for the JSON format. This release introduces a new -item reader that can read a JSON resource in the following format: - -[source, json] ----- -[ - { - "isin": "123", - "quantity": 1, - "price": 1.2, - "customer": "foo" - }, - { - "isin": "456", - "quantity": 2, - "price": 1.4, - "customer": "bar" - } -] ----- - -Similar to the `StaxEventItemReader` for XML, the new `JsonItemReader` uses streaming -APIs to read JSON objects in chunks. Spring Batch supports two libraries: - -* link:$$https://github.com/FasterXML/jackson$$[Jackson] -* link:$$https://github.com/google/gson$$[Gson] - -To add other libraries, you can implement the `JsonObjectReader` interface. - -Writing JSON data is also supported through the `JsonFileItemWriter`. -For more details about JSON support, see the -<> chapter. - -[[whatsNewBeanValidationApi]] -=== Bean Validation API support - -This release brings a new `ValidatingItemProcessor` implementation called -`BeanValidatingItemProcessor` which allows you to validate items annotated with -the Bean Validation API (JSR-303) annotations. For example, given the following -type `Person`: - -[source, java] ----- -class Person { - - @NotEmpty - private String name; - - public Person(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - -} ----- - -you can validate items by declaring a `BeanValidatingItemProcessor` bean in your -application context and register it as a processor in your chunk-oriented step: - -[source, java] ----- -@Bean -public BeanValidatingItemProcessor beanValidatingItemProcessor() throws Exception { - BeanValidatingItemProcessor beanValidatingItemProcessor = new BeanValidatingItemProcessor<>(); - beanValidatingItemProcessor.setFilter(true); - - return beanValidatingItemProcessor; -} ----- - -[[whatsNewJSR305Api]] -=== JSR-305 support - -This release adds support for JSR-305 annotations. It leverages Spring Framework’s -link:$$https://docs.spring.io/spring/docs/current/spring-framework-reference/core.html#null-safety$$[Null-safety] -annotations and adds them on all public APIs of Spring Batch. - -These annotations will not only enforce null-safety when using Spring Batch APIs, -but also can be used by IDEs to provide useful information related to nullability. -For example, if a user wants to implement the `ItemReader` interface, any IDE -supporting JSR-305 annotations will generate something like: - -[source, java] ----- -public class MyItemReader implements ItemReader { - - @Nullable - public String read() throws Exception { - return null; - } - -} ----- - -The `@Nullable` annotation present on the `read` method makes it clear that the -contract of this method says it may return `null`. This enforces what is said in its -Javadoc, that the `read` method should return `null` when the data source is exhausted. - -[[whatsNewFlatFileItemWriterBuilder]] -=== `FlatFileItemWriterBuilder` enhancements - -Another small feature added in this release is a simplification of the configuration -for the writing of a flat file. Specifically, these updates simplify the configuration -of both a delimited and fixed width file. Below is an example of before and after the change. - -[source, java] ----- -// Before -@Bean -public FlatFileItemWriter itemWriter(Resource resource) { - BeanWrapperFieldExtractor fieldExtractor = - new BeanWrapperFieldExtractor(); - fieldExtractor.setNames(new String[] {"field1", "field2", "field3"}); - fieldExtractor.afterPropertiesSet(); - - DelimitedLineAggregator aggregator = new DelimitedLineAggregator(); - aggregator.setFieldExtractor(fieldExtractor); - aggregator.setDelimiter(";"); - - return new FlatFileItemWriterBuilder() - .name("itemWriter") - .resource(resource) - .lineAggregator(aggregator) - .build(); -} - -// After -@Bean -public FlatFileItemWriter itemWriter(Resource resource) { - return new FlatFileItemWriterBuilder() - .name("itemWriter") - .resource(resource) - .delimited() - .delimiter(";") - .names(new String[] {"field1", "field2", "field3"}) - .build(); -} ----- +The reference documentation has been updated to match the same style as other +Spring projects. diff --git a/spring-batch-samples/README.md b/spring-batch-samples/README.md index 5a77f34f8..56111c6fa 100644 --- a/spring-batch-samples/README.md +++ b/spring-batch-samples/README.md @@ -888,3 +888,34 @@ file to another. It uses XStream for the object XML conversion, because this is simple to configure for basic use cases like this one. See [Spring OXM documentation](https://docs.spring.io/spring/docs/current/spring-framework-reference/data-access.html#oxm) for details of other options. + +### Batch metrics with Micrometer + +This sample shows how to use [Micrometer](https://micrometer.io) to collect batch metrics in Spring Batch. +It uses [Prometheus](https://prometheus.io) as the metrics back end and [Grafana](https://grafana.com) as the front end. +The sample consists of two jobs: + +* `job1` : Composed of two tasklets that print `hello` and `world` +* `job2` : Composed of single chunk-oriented step that reads and writes a random number of items + +These two jobs are run repeatedly at regular intervals and might fail randomly for demonstration purposes. + +This sample requires [docker compose](https://docs.docker.com/compose/) to start the monitoring stack. +To run the sample, please follow these steps: + +``` +$>cd spring-batch-samples/src/grafana +$>docker-compose up -d +``` + +This should start the required monitoring stack: + +* Prometheus server on port `9090` +* Prometheus push gateway on port `9091` +* Grafana on port `3000` + +Once started, you need to [configure Prometheus as data source in Grafana](https://grafana.com/docs/features/datasources/prometheus/) +and import the ready-to-use dashboard in `spring-batch-samples/src/grafana/spring-batch-dashboard.json`. + +Finally, run the `org.springframework.batch.sample.metrics.BatchMetricsApplication` +class without any argument to start the sample. diff --git a/spring-batch-samples/src/grafana/docker-compose.yml b/spring-batch-samples/src/grafana/docker-compose.yml new file mode 100644 index 000000000..f82d08a70 --- /dev/null +++ b/spring-batch-samples/src/grafana/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3.3' +services: + + prometheus: + image: prom/prometheus:v2.7.2 + container_name: 'prometheus' + ports: + - '9090:9090' + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + + pushgateway: + image: prom/pushgateway:v0.6.0 + container_name: 'pushgateway' + ports: + - '9091:9091' + + grafana: + image: grafana/grafana:6.0.2 + container_name: 'grafana' + ports: + - '3000:3000' diff --git a/spring-batch-samples/src/grafana/prometheus/prometheus.yml b/spring-batch-samples/src/grafana/prometheus/prometheus.yml new file mode 100644 index 000000000..4c469dd3d --- /dev/null +++ b/spring-batch-samples/src/grafana/prometheus/prometheus.yml @@ -0,0 +1,9 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: 'springbatch' + honor_labels: true + static_configs: + - targets: ['pushgateway:9091'] diff --git a/spring-batch-samples/src/grafana/spring-batch-dashboard.json b/spring-batch-samples/src/grafana/spring-batch-dashboard.json new file mode 100644 index 000000000..8660bbb2e --- /dev/null +++ b/spring-batch-samples/src/grafana/spring-batch-dashboard.json @@ -0,0 +1,568 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "description": "Dashboard for Spring Batch applications instrumented with Micrometer", + "editable": true, + "gnetId": 4701, + "graphTooltip": 1, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 132, + "panels": [], + "repeat": null, + "title": "Spring Batch Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 37, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "spring_batch_job_seconds_max", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}} {{status}}", + "metric": "", + "refId": "A", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Job duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(spring_batch_job_seconds_count[1m]) * 60", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} {{status}}", + "metric": "", + "refId": "A", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Job execution rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ops", + "short" + ], + "yaxes": [ + { + "decimals": null, + "format": "opm", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 138, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(spring_batch_job_seconds_count{status=\"FAILED\"}[5m])) by (name, status)) / (sum(rate(spring_batch_job_seconds_count[5m])) by (name, status))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}} {{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Job failure rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 141, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "write success", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(spring_batch_item_read_seconds_count[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{status}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Item read throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 140, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "write success", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(spring_batch_chunk_write_seconds_count[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{status}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Item write throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Spring Batch Prometheus", + "uid": "qRLUmOCmk", + "version": 6 +} diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/BatchMetricsApplication.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/BatchMetricsApplication.java new file mode 100644 index 000000000..eb6e2aa23 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/BatchMetricsApplication.java @@ -0,0 +1,31 @@ +package org.springframework.batch.sample.metrics; + +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; + +@EnableScheduling +@EnableBatchProcessing +@Import({Job1Configuration.class, Job2Configuration.class, JobScheduler.class, PrometheusConfiguration.class}) +@PropertySource("metrics-sample.properties") +public class BatchMetricsApplication { + + public static void main(String[] args) { + AnnotationConfigApplicationContext applicationContext = new AnnotationConfigApplicationContext(BatchMetricsApplication.class); + applicationContext.start(); + } + + @Bean(destroyMethod = "shutdown") + public ThreadPoolTaskScheduler taskScheduler(@Value("${thread.pool.size}") int threadPoolSize) { + ThreadPoolTaskScheduler threadPoolTaskScheduler = new ThreadPoolTaskScheduler(); + threadPoolTaskScheduler.setPoolSize(threadPoolSize); + return threadPoolTaskScheduler; + } + +} + diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/Job1Configuration.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/Job1Configuration.java new file mode 100644 index 000000000..406c4dbc2 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/Job1Configuration.java @@ -0,0 +1,62 @@ +package org.springframework.batch.sample.metrics; + +import java.util.Random; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class Job1Configuration { + + private Random random; + private JobBuilderFactory jobs; + private StepBuilderFactory steps; + + public Job1Configuration(JobBuilderFactory jobs, StepBuilderFactory steps) { + this.jobs = jobs; + this.steps = steps; + this.random = new Random(); + } + + @Bean + public Job job1() { + return jobs.get("job1") + .start(step1()) + .next(step2()) + .build(); + } + + @Bean + public Step step1() { + return steps.get("step1") + .tasklet((contribution, chunkContext) -> { + System.out.println("hello"); + // simulate processing time + Thread.sleep(random.nextInt(3000)); + return RepeatStatus.FINISHED; + }) + .build(); + } + + @Bean + public Step step2() { + return steps.get("step2") + .tasklet((contribution, chunkContext) -> { + System.out.println("world"); + // simulate step failure + int nextInt = random.nextInt(3000); + Thread.sleep(nextInt); + if (nextInt % 5 == 0) { + throw new Exception("Boom!"); + } + return RepeatStatus.FINISHED; + }) + .build(); + } + +} diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/Job2Configuration.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/Job2Configuration.java new file mode 100644 index 000000000..fe1289109 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/Job2Configuration.java @@ -0,0 +1,72 @@ +package org.springframework.batch.sample.metrics; + +import java.util.LinkedList; +import java.util.List; +import java.util.Random; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.support.ListItemReader; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class Job2Configuration { + + private Random random; + private JobBuilderFactory jobs; + private StepBuilderFactory steps; + + public Job2Configuration(JobBuilderFactory jobs, StepBuilderFactory steps) { + this.jobs = jobs; + this.steps = steps; + this.random = new Random(); + } + + @Bean + public Job job2() { + return jobs.get("job2") + .start(step()) + .build(); + } + + @Bean + public Step step() { + return steps.get("step1") + .chunk(3) + .reader(itemReader()) + .writer(itemWriter()) + .build(); + } + + @Bean + @StepScope + public ListItemReader itemReader() { + List items = new LinkedList<>(); + // read a random number of items in each run + for (int i = 0; i < random.nextInt(100); i++) { + items.add(i); + } + return new ListItemReader<>(items); + } + + @Bean + public ItemWriter itemWriter() { + return items -> { + for (Integer item : items) { + int nextInt = random.nextInt(1000); + Thread.sleep(nextInt); + // simulate write failure + if (nextInt % 57 == 0) { + throw new Exception("Boom!"); + } + System.out.println("item = " + item); + } + }; + } + +} diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/JobScheduler.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/JobScheduler.java new file mode 100644 index 000000000..43382b1f7 --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/JobScheduler.java @@ -0,0 +1,43 @@ +package org.springframework.batch.sample.metrics; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +public class JobScheduler { + + private final Job job1; + private final Job job2; + private final JobLauncher jobLauncher; + + @Autowired + public JobScheduler(Job job1, Job job2, JobLauncher jobLauncher) { + this.job1 = job1; + this.job2 = job2; + this.jobLauncher = jobLauncher; + } + + @Scheduled(cron="*/10 * * * * *") + public void launchJob1() throws Exception { + JobParameters jobParameters = new JobParametersBuilder() + .addLong("time", System.currentTimeMillis()) + .toJobParameters(); + + jobLauncher.run(job1, jobParameters); + } + + @Scheduled(cron="*/15 * * * * *") + public void launchJob2() throws Exception { + JobParameters jobParameters = new JobParametersBuilder() + .addLong("time", System.currentTimeMillis()) + .toJobParameters(); + + jobLauncher.run(job2, jobParameters); + } + +} diff --git a/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/PrometheusConfiguration.java b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/PrometheusConfiguration.java new file mode 100644 index 000000000..b073a1cbf --- /dev/null +++ b/spring-batch-samples/src/main/java/org/springframework/batch/sample/metrics/PrometheusConfiguration.java @@ -0,0 +1,55 @@ +package org.springframework.batch.sample.metrics; + +import java.util.HashMap; +import java.util.Map; +import javax.annotation.PostConstruct; + +import io.micrometer.core.instrument.Metrics; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.PushGateway; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.Scheduled; + +@Configuration +public class PrometheusConfiguration { + + private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusConfiguration.class); + + @Value("${prometheus.job.name}") + private String prometheusJobName; + + @Value("${prometheus.grouping.key}") + private String prometheusGroupingKey; + + @Value("${prometheus.pushgateway.url}") + private String prometheusPushGatewayUrl; + + private Map groupingKey = new HashMap<>(); + private PushGateway pushGateway; + private CollectorRegistry collectorRegistry; + + @PostConstruct + public void init() { + pushGateway = new PushGateway(prometheusPushGatewayUrl); + groupingKey.put(prometheusGroupingKey, prometheusJobName); + PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + collectorRegistry = prometheusMeterRegistry.getPrometheusRegistry(); + Metrics.globalRegistry.add(prometheusMeterRegistry); + } + + @Scheduled(fixedRateString = "${prometheus.push.rate}") + public void pushMetrics() { + try { + pushGateway.pushAdd(collectorRegistry, prometheusJobName, groupingKey); + } catch (Throwable ex) { + LOGGER.error("Unable to push metrics to Prometheus Push Gateway", ex); + } + } + +} diff --git a/spring-batch-samples/src/main/resources/metrics-sample.properties b/spring-batch-samples/src/main/resources/metrics-sample.properties new file mode 100644 index 000000000..2c30f3a8b --- /dev/null +++ b/spring-batch-samples/src/main/resources/metrics-sample.properties @@ -0,0 +1,5 @@ +thread.pool.size=3 +prometheus.push.rate=5000 +prometheus.job.name=springbatch +prometheus.grouping.key=appname +prometheus.pushgateway.url=0.0.0.0:9091