diff --git a/monitoring-samples/stream-apps/SCDF-monitoring-promethesu-proxy.gif b/monitoring-samples/stream-apps/SCDF-monitoring-promethesu-proxy.gif new file mode 100644 index 0000000..1d1145a Binary files /dev/null and b/monitoring-samples/stream-apps/SCDF-monitoring-promethesu-proxy.gif differ diff --git a/src/main/asciidoc/micrometer/influx/main.adoc b/src/main/asciidoc/micrometer/influx/main.adoc deleted file mode 100644 index 98a9cc6..0000000 --- a/src/main/asciidoc/micrometer/influx/main.adoc +++ /dev/null @@ -1,14 +0,0 @@ -[[spring-cloud-data-flow-samples-micrometer-influx-overview]] -:sectnums: -:docs_dir: ../.. - -=== SCDF metrics with InfluxDB and Grafana - -Demonstration, how to monitor your https://cloud.spring.io/spring-cloud-dataflow/[Spring Cloud Data Flow] (SCDF) streams using https://docs.influxdata.com/influxdb/v1.5/[InfluxDB] and https://grafana.com/grafana[Grafana]. - -By default the https://docs.spring.io/spring-cloud-dataflow/docs/2.0.0.BUILD-SNAPSHOT/reference/htmlsingle/#getting-started-local-deploying-spring-cloud-dataflow-docker[Data Flow 2.0+ docker-compose configures] Stream monitoring with InfluxDB and pre-built dashboards for Grafana. - - -For further instructions about Data Flow monitoring, follow the https://docs.spring.io/spring-cloud-dataflow/docs/2.0.0.BUILD-SNAPSHOT/reference/htmlsingle/#streams-monitoring-local[Streams Monitoring InfluxDB] section. - -image::grafana-influxdb-scdf-streams-dashboard.png[Grafana InfluxDB Dashboard, scaledwidth="50%"] diff --git a/src/main/asciidoc/micrometer/influx/scdf-influxdb-dashboard.json b/src/main/asciidoc/micrometer/influx/scdf-influxdb-dashboard.json deleted file mode 100644 index ddf47b5..0000000 --- a/src/main/asciidoc/micrometer/influx/scdf-influxdb-dashboard.json +++ /dev/null @@ -1,1688 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR", - "label": "influx_auto_DataFlowMetricsCollector", - "description": "", - "type": "datasource", - "pluginId": "influxdb", - "pluginName": "InfluxDB" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.1.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1525333319927, - "links": [ - { - "icon": "external link", - "tags": [], - "type": "dashboards" - } - ], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11, - "panels": [], - "repeat": null, - "title": "Input Channel", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "application_name" - ], - "type": "tag" - }, - { - "params": [ - "result" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - } - ], - "measurement": "spring_integration_send", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "count" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "type", - "operator": "=", - "value": "channel" - }, - { - "condition": "AND", - "key": "name", - "operator": "=", - "value": "input" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Input Channel Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - }, - { - "params": [ - "result" - ], - "type": "tag" - } - ], - "measurement": "spring_integration_send", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - }, - { - "condition": "AND", - "key": "type", - "operator": "=", - "value": "channel" - }, - { - "condition": "AND", - "key": "name", - "operator": "=", - "value": "input" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Input Channel Average Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 12, - "panels": [], - "repeat": null, - "title": "Output Channel", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - }, - { - "params": [ - "result" - ], - "type": "tag" - } - ], - "measurement": "spring_integration_send", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT \"count\" FROM \"spring_integration_send\" WHERE (\"application_name\" =~ /^$application$/ AND \"type\" = 'channel' AND \"name\" = 'output') AND $timeFilter GROUP BY \"stream_name\", \"application_name\", \"result\"", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "count" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - }, - { - "condition": "AND", - "key": "type", - "operator": "=", - "value": "channel" - }, - { - "condition": "AND", - "key": "name", - "operator": "=", - "value": "output" - }, - { - "condition": "AND", - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Output Channel Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - }, - { - "params": [ - "result" - ], - "type": "tag" - } - ], - "measurement": "spring_integration_send", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT \"mean\" FROM \"spring_integration_send\" WHERE (\"stream_name\" =~ /^$stream$/ AND \"application_name\" =~ /^$application$/ AND \"type\" = 'channel' AND \"name\" = 'output') AND $timeFilter GROUP BY \"stream_name\", \"application_name\", \"result\"", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - }, - { - "condition": "AND", - "key": "type", - "operator": "=", - "value": "channel" - }, - { - "condition": "AND", - "key": "name", - "operator": "=", - "value": "output" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Output Channel Average Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 13, - "panels": [], - "repeat": null, - "title": "Error Channel", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - }, - { - "params": [ - "result" - ], - "type": "tag" - } - ], - "measurement": "spring_integration_send", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "count" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - }, - { - "condition": "AND", - "key": "type", - "operator": "=", - "value": "channel" - }, - { - "condition": "AND", - "key": "name", - "operator": "=", - "value": "errorChannel" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Error Channel Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - }, - { - "params": [ - "result" - ], - "type": "tag" - } - ], - "measurement": "spring_integration_send", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - }, - { - "condition": "AND", - "key": "type", - "operator": "=", - "value": "channel" - }, - { - "condition": "AND", - "key": "name", - "operator": "=", - "value": "errorChannel" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Error Channel Average Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 14, - "panels": [], - "repeat": null, - "title": "Handlers", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - }, - { - "params": [ - "result" - ], - "type": "tag" - }, - { - "params": [ - "name" - ], - "type": "tag" - } - ], - "measurement": "spring_integration_send", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "count" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - }, - { - "condition": "AND", - "key": "type", - "operator": "=", - "value": "handler" - }, - { - "condition": "AND", - "key": "name", - "operator": "!=", - "value": "org_springframework_cloud_stream_app_log_sink_LogSinkConfiguration_logSinkHandler_serviceActivator_handler" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Handlers Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - }, - { - "params": [ - "result" - ], - "type": "tag" - }, - { - "params": [ - "name" - ], - "type": "tag" - } - ], - "measurement": "spring_integration_send", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - }, - { - "condition": "AND", - "key": "type", - "operator": "=", - "value": "handler" - }, - { - "condition": "AND", - "key": "name", - "operator": "!=", - "value": "org_springframework_cloud_stream_app_log_sink_LogSinkConfiguration_logSinkHandler_serviceActivator_handler" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Handlers Average Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 15, - "panels": [], - "repeat": null, - "title": "Dashboard Row", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 9, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [], - "measurement": "process_cpu_usage", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [ - " * 100" - ], - "type": "math" - } - ] - ], - "tags": [] - }, - { - "dsType": "influxdb", - "groupBy": [], - "measurement": "system_cpu_usage", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [ - " * 100" - ], - "type": "math" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 10, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "1m" - ], - "type": "time" - }, - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - } - ], - "measurement": "jvm_memory_used", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - } - ] - }, - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "1m" - ], - "type": "time" - }, - { - "params": [ - "stream_name" - ], - "type": "tag" - }, - { - "params": [ - "application_name" - ], - "type": "tag" - } - ], - "hide": false, - "measurement": "jvm_memory_max", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "stream_name", - "operator": "=~", - "value": "/^$stream$/" - }, - { - "condition": "AND", - "key": "application_name", - "operator": "=~", - "value": "/^$application$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "JVM Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "hide": 0, - "includeAll": false, - "label": "Stream", - "multi": false, - "name": "stream", - "options": [], - "query": "show tag values with key=\"stream_name\"", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "hide": 0, - "includeAll": false, - "label": "application", - "multi": true, - "name": "application", - "options": [], - "query": "show tag values with key=\"application_name\" WHERE stream_name='$stream'", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "show tag values with key=\"application_name\" WHERE stream='$stream' AND stream_type = '$tag'", - "tags": [], - "tagsQuery": "show tag values with key = \"stream_type\"", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_INFLUX_AUTO_DATAFLOWMETRICSCOLLECTOR}", - "hide": 0, - "includeAll": false, - "label": "ApplicationId", - "multi": false, - "name": "application_id", - "options": [], - "query": "show tag values from \"spring_integration_send\" with key = \"application_guid\" WHERE stream_name = '$stream'", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "show tag values with key = \"application_guid\" WHERE stream_name = '$stream' and application_name = '$tag'", - "tags": [ - "log2", - "time2" - ], - "tagsQuery": "show tag values with key=\"application_name\" WHERE stream_name='$stream'", - "type": "query", - "useTags": true - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "scdf-influxdb-dashboard", - "uid": "QvcTSLGik", - "version": 1 -} diff --git a/src/main/asciidoc/micrometer/prometheus/grafana-twitter-scdf-analytics.json b/src/main/asciidoc/micrometer/prometheus/grafana-twitter-scdf-analytics.json deleted file mode 100644 index d8f9919..0000000 --- a/src/main/asciidoc/micrometer/prometheus/grafana-twitter-scdf-analytics.json +++ /dev/null @@ -1,298 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_SCDFPROMETHEUS", - "label": "ScdfPrometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "panel", - "id": "digrich-bubblechart-panel", - "name": "Bubble Chart", - "version": "1.1.0" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "6.3.3" - }, - { - "type": "panel", - "id": "grafana-piechart-panel", - "name": "Pie Chart", - "version": "1.3.9" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "links": [ - { - "icon": "external link", - "tags": [], - "type": "dashboards" - } - ], - "panels": [ - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_SCDFPROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 15, - "x": 0, - "y": 0 - }, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sort_desc(topk(10, sum(language_total) by (lang)))", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{lang}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Top 10 Lang", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "series", - "name": null, - "show": true, - "values": [ - "current" - ] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": "${DS_SCDFPROMETHEUS}", - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 22, - "w": 9, - "x": 15, - "y": 0 - }, - "id": 4, - "interval": null, - "legend": { - "show": true, - "values": true - }, - "legendType": "Under graph", - "links": [], - "maxDataPoints": 3, - "nullPointMode": "connected", - "options": {}, - "pieType": "donut", - "strokeWidth": 1, - "targets": [ - { - "expr": "sort_desc(topk(50, sum(hashtags_total) by (htag)))", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{htag}}", - "refId": "A" - } - ], - "title": "Hastags", - "type": "grafana-piechart-panel", - "valueName": "current" - }, - { - "bgColor": null, - "colorScheme": "Unique", - "datasource": "${DS_SCDFPROMETHEUS}", - "decimal": 2, - "displayLabel": true, - "format": "short", - "gradientColors": [ - "red", - "green" - ], - "gradientThresholds": "50,80", - "gridPos": { - "h": 12, - "w": 15, - "x": 0, - "y": 10 - }, - "groupDepthColors": [ - "hsl(152,80%,80%)", - "hsl(228,30%,40%)" - ], - "groupSeperator": ",", - "height": 400, - "id": 2, - "links": [], - "mode": "time", - "nullPointMode": "connected", - "options": {}, - "svgBubbleId": "svg_2", - "svgContainer": {}, - "targets": [ - { - "expr": "sum(language_total) by (lang)", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{lang}}", - "refId": "A" - } - ], - "thresholdColors": [ - "green", - "yellow", - "red" - ], - "thresholds": "50,80", - "title": "Languages", - "type": "digrich-bubblechart-panel", - "valueName": "current" - } - ], - "refresh": "5s", - "schemaVersion": 19, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "SCDF Analytics", - "uid": "vhHweSriz", - "version": 3 -} diff --git a/src/main/asciidoc/micrometer/prometheus/main.adoc b/src/main/asciidoc/micrometer/prometheus/main.adoc deleted file mode 100644 index 9d2aceb..0000000 --- a/src/main/asciidoc/micrometer/prometheus/main.adoc +++ /dev/null @@ -1,23 +0,0 @@ -[[spring-cloud-data-flow-samples-micrometer-prometheus-overview]] -:sectnums: -:docs_dir: ../.. - -=== SCDF metrics with Prometheus and Grafana - -In this demonstration, you will learn how to monitor your https://cloud.spring.io/spring-cloud-dataflow/[Spring Cloud Data Flow] Streams using https://prometheus.io[Prometheus] and https://grafana.com/grafana[Grafana]. - -Starting with 2.0, Data Flow provides built-in support for monitoring with Prometheus. -Following the reference manual instructions for how to get started with Prometheus: - -* https://docs.spring.io/spring-cloud-dataflow/docs/2.0.0.BUILD-SNAPSHOT/reference/htmlsingle/#streams-monitoring-local-prometheus[Prometheus with SCDF-Local] -* https://docs.spring.io/spring-cloud-dataflow/docs/2.0.0.BUILD-SNAPSHOT/reference/htmlsingle/#streams-monitoring-kubernetes-prometheus[Prometheus with SCDF-Kubernetes] - -Following diagram (Fig.4) illustrates the metrics collection flows for monitoring with prometheus on a Local SCDF platform: - -image::scdf-micrometer-prometheus-grafana-architecture.png[title="SCDF metrics analyzis with Prometheus and Grafana", scaledwidth="50%"] - -Out of the box, SCSt sends https://docs.spring.io/spring-boot/docs/2.0.1.RELEASE/reference/htmlsingle/#production-ready-metrics-meter[core metrics] such as `CPU`, `Memory`, `MVC` and `Health` to name some. Among those the https://docs.spring.io/spring-integration/docs/current/reference/html/system-management-chapter.html#micrometer-integration[Spring Integration metrics] allows computing the `Rate` and the `Latency` of the messages in the SCDF streams. - -Once you deploy the stream you should see dashboards similar to these. - -image:https://raw.githubusercontent.com/spring-cloud/spring-cloud-dataflow/master/spring-cloud-dataflow-docs/src/main/asciidoc/images/grafana-prometheus-scdf-applications-dashboard.png[SCDF Prometheus Dashboard] diff --git a/src/main/asciidoc/micrometer/prometheus/scdf-prometheus-grafana-dashboard.json b/src/main/asciidoc/micrometer/prometheus/scdf-prometheus-grafana-dashboard.json deleted file mode 100644 index 9317e42..0000000 --- a/src/main/asciidoc/micrometer/prometheus/scdf-prometheus-grafana-dashboard.json +++ /dev/null @@ -1,718 +0,0 @@ - -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": 2, - "links": [ - { - "icon": "external link", - "tags": [], - "type": "dashboards" - } - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": 282, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "ScdfPrometheus", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(spring_integration_send_seconds_sum{type=\"channel\",name!=\"errorChannel\",name!=\"nullChannel\",result=\"success\",stream_name=\"$stream_name\",application_name=~\"$application_name\",application_guid=~\"$application_guid\"}[60s])/rate(spring_integration_send_seconds_count{type=\"channel\",name!=\"errorChannel\",name!=\"nullChannel\",result=\"success\",stream_name=\"$stream_name\",application_name=~\"$application_name\"}[60s])", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Channel Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "ScdfPrometheus", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(spring_integration_send_seconds_count{type=\"channel\",name!=\"errorChannel\",name!=\"nullChannel\",result=\"success\",stream_name=~\"$stream_name\",application_name=~\"$application_name\",application_guid=~\"$application_guid\"}[60s])", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Channel Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Channels (success)", - "titleSize": "h4" - }, - { - "collapse": false, - "height": 281, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "ScdfPrometheus", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(spring_integration_send_seconds_sum{type=\"channel\",result=\"failure\",name!=\"errorChannel\",name!=\"nullChannel\",stream_name=\"$stream_name\",application_name=~\"$application_name\",application_guid=~\"$application_guid\"}[60s])/rate(spring_integration_send_seconds_count{type=\"channel\",result=\"failure\",name!=\"errorChannel\",name!=\"nullChannel\",stream_name=\"$stream_name\",application_name=~\"$application_name\"}[60s])", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Channel Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "ScdfPrometheus", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(spring_integration_send_seconds_count{type=\"channel\",result=\"failure\",name!=\"errorChannel\",name!=\"nullChannel\",stream_name=~\"$stream_name\",application_name=~\"$application_name\",application_guid=~\"$application_guid\"}[60s])", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Channel Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Channels (failure)", - "titleSize": "h4" - }, - { - "collapse": true, - "height": 250, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "ScdfPrometheus", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(spring_integration_send_seconds_sum{type=\"handler\",result=\"success\",stream_name=\"$stream_name\",application_name=~\"$application_name\",application_guid=~\"$application_guid\"}[60s])/rate(spring_integration_send_seconds_count{type=\"handler\",result=\"success\",stream_name=\"$stream_name\",application_name=~\"$application_name\"}[60s])", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Handler Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "ScdfPrometheus", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(spring_integration_send_seconds_count{type=\"handler\",result=\"success\",stream_name=~\"$stream_name\",application_name=~\"$application_name\",application_guid=~\"$application_guid\"}[60s])", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Handler Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Handlers", - "titleSize": "h4" - }, - { - "collapse": true, - "height": 250, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "ScdfPrometheus", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(spring_integration_send_seconds_count{type=\"channel\",name=\"errorChannel\",stream_name=~\"$stream_name\",application_name=~\"$application_name\",application_guid=~\"$application_guid\"}[60s])", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Error Channel Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Error Channels", - "titleSize": "h4" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "text": "t2", - "value": "t2" - }, - "datasource": "ScdfPrometheus", - "hide": 0, - "includeAll": false, - "label": "Stream", - "multi": false, - "name": "stream_name", - "options": [], - "query": "label_values(stream_name)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "time2 + log2", - "value": [ - "time2", - "log2" - ] - }, - "datasource": "ScdfPrometheus", - "hide": 0, - "includeAll": true, - "label": "Application", - "multi": true, - "name": "application_name", - "options": [], - "query": "label_values({stream_name=\"$stream_name\"},application_name)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "8082", - "value": "8082" - }, - "datasource": "ScdfPrometheus", - "hide": 0, - "includeAll": false, - "label": "ApplicationGuid", - "multi": true, - "name": "application_guid", - "options": [], - "query": "label_values({stream_name=~\"$stream_name\",application_name=~\"$application_name\"},application_guid)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "label_values({stream_name=~\"$stream_name\",application_name=~\"$tag\"},application_guid)", - "tags": [ - "time2", - "log2" - ], - "tagsQuery": "label_values({stream_name=~\"$stream_name\"},application_name)", - "type": "query", - "useTags": true - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "SCDF - Prometheus Dashboard", - "version": 19 -} diff --git a/src/main/asciidoc/micrometer/prometheus/spring-cloud-dataflow-prometheus-service-discovery-0.0.1-SNAPSHOT.jar b/src/main/asciidoc/micrometer/prometheus/spring-cloud-dataflow-prometheus-service-discovery-0.0.1-SNAPSHOT.jar deleted file mode 100644 index 3b8216f..0000000 Binary files a/src/main/asciidoc/micrometer/prometheus/spring-cloud-dataflow-prometheus-service-discovery-0.0.1-SNAPSHOT.jar and /dev/null differ diff --git a/src/main/asciidoc/monitoring/main.adoc b/src/main/asciidoc/monitoring/main.adoc new file mode 100644 index 0000000..8ef07e5 --- /dev/null +++ b/src/main/asciidoc/monitoring/main.adoc @@ -0,0 +1,23 @@ +[[spring-cloud-data-flow-samples-monitoring]] +:sectnums: +:docs_dir: .. + +=== Spring Cloud Data Flow Monitoring + +The https://dataflow.spring.io/docs/2.3.0.SNAPSHOT/concepts/monitoring/[Data Flow Monitoring Architecture] is designed around the Micrometer library and is configured to support two of the most popular monitoring systems, https://docs.influxdata.com/influxdb/v1.5/[InfluxDB], https://prometheus.io[Prometheus]. +To help you get started monitoring Streams and Tasks, Data Flow provides https://grafana.com/[Grafana Dashboards] that you can customize for your needs. + +image:https://raw.githubusercontent.com/spring-cloud/spring-cloud-dataflow-samples/master/monitoring-samples/images/SCDF-monitoring-promethesu-proxy.gif[SCDF Stream Monitoring, scaledwidth="100%"] + +Following links will provide you with information about installing, enabling and using the monitoring across the Local, Kubernetes and Cloud Foundry platforms for Streams and Tasks: + +- https://dataflow.spring.io/docs/2.3.0.SNAPSHOT/feature-guides/streams/monitoring/[Stream Monitoring with Prometheus and InfluxDB] +- https://dataflow.spring.io/docs/2.3.0.SNAPSHOT/feature-guides/batch/monitoring/#task-and-batch-monitoring-with-prometheus-and-influxdb[Task and Batch Monitoring with Prometheus and InfluxDB] + +All provided https://cloud.spring.io/spring-cloud-stream-app-starters/[Stream application starters] are configured for Prometheus and InfluxDB. + +To integrate custom applications with the Data Flow Monitoring you can follow the https://github.com/spring-cloud/spring-cloud-dataflow-samples/tree/master/monitoring-samples[monitoring-samples]: +- The https://github.com/spring-cloud/spring-cloud-dataflow-samples/tree/master/monitoring-samples/stream-apps[stream-apps] shows how to enable monitoring for custom built source, processor and sink apps. +- The https://github.com/spring-cloud/spring-cloud-dataflow-samples/tree/master/monitoring-samples/task-apps[task-apps] sample shows how to enable monitoring for custom built task apps. + + diff --git a/src/main/asciidoc/overview.adoc b/src/main/asciidoc/overview.adoc index 7a312c5..30adb05 100644 --- a/src/main/asciidoc/overview.adoc +++ b/src/main/asciidoc/overview.adoc @@ -29,28 +29,8 @@ include::datascience/species-prediction/main.adoc[] == Functions include::functions/main.adoc[] -== Monitoring (Micrometer) -Demonstrate how to monitor your https://cloud.spring.io/spring-cloud-dataflow/[Spring Cloud Data Flow] (SCDF) streams using https://docs.influxdata.com/influxdb/v1.5/[InfluxDB], https://prometheus.io[Prometheus] and https://grafana.com/grafana[Grafana]. - -The https://docs.spring.io/spring-cloud-dataflow/docs/2.0.0.BUILD-SNAPSHOT/reference/htmlsingle/#streams-monitoring[Data Flow 2.x metrics architecture] is designed around the https://micrometer.io/[Micrometer library] and provides support for two of the most popular monitoring systems, Prometheus and InfluxDB. -Also to help you get started monitoring Streams, Data Flow provides https://grafana.com/[Grafana Dashboards] you can install and customize for your needs. -Support for monitoring Tasks is on the roadmap. - -The general architecture of how applications are monitored is shown below. - -image:https://raw.githubusercontent.com/spring-cloud/spring-cloud-dataflow/master/spring-cloud-dataflow-docs/src/main/asciidoc/images/micrometer-arch.png[SCDF Stream Monitoring, scaledwidth="100%"] - -You can find more information in the https://docs.spring.io/spring-cloud-dataflow/docs/2.0.0.BUILD-SNAPSHOT/reference/htmlsingle/#streams-monitoring[SCDF Stream Monitoring] reference manual. - -* https://prometheus.io[Prometheus] - pull based time series database used for monitoring of highly dynamic service-oriented architectures. -In a world of microservices, its support for multi-dimensional data collection and querying is a particular strength. -* https://github.com/influxdata/influxdb[InfluxDB] is a popular open-source push based time series database. -It supports downsampling, automatically expiring and deleting unwanted data, as well as backup and restore. Analysis of data is done via a https://docs.influxdata.com/influxdb/v1.5/query_language/[SQL-like query] language. -* https://grafana.com/grafana[Grafana] is open source metrics Dashboard platform. It supports multiple backend time-series databases including InluxDB and Prometheus. - -include::micrometer/influx/main.adoc[] -include::micrometer/prometheus/main.adoc[] - +== Monitoring +include::monitoring/main.adoc[] == Security include::security/main.adoc[]