From 0231b2bbf0b0a2cf95b255839c79bbbbc28527e7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 14:34:54 +0300 Subject: [PATCH] [#38] grafana: Add blobstore dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 565 +++++++++++++++++- 1 file changed, 551 insertions(+), 14 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index a2a37aa..0d8213f 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -168,10 +168,6 @@ "description": "Current epoch on the instance", "fieldConfig": { "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, "mappings": [], "thresholds": { "mode": "absolute", @@ -186,7 +182,7 @@ } ] }, - "unit": "none" + "unit": "short" }, "overrides": [] }, @@ -198,6 +194,9 @@ }, "id": 32, "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -206,8 +205,7 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "textMode": "auto" }, "pluginVersion": "9.5.6", "targets": [ @@ -224,7 +222,7 @@ } ], "title": "Epoch", - "type": "gauge" + "type": "stat" }, { "collapsed": true, @@ -1066,7 +1064,7 @@ "h": 10, "w": 24, "x": 0, - "y": 10 + "y": 51 }, "id": 35, "options": { @@ -1167,7 +1165,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 61 }, "id": 37, "options": { @@ -1267,7 +1265,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 71 }, "id": 36, "options": { @@ -1369,7 +1367,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 81 }, "id": 34, "options": { @@ -2138,6 +2136,545 @@ "x": 0, "y": 11 }, + "id": 38, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobstore method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobstore requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobstore_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobstore shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobstore mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobstore_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobstore", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, "id": 23, "panels": [ { @@ -2450,7 +2987,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 6, "panels": [ @@ -3000,7 +3537,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" },