From d8df46b4d19ae673d44a0e55585c1a3fa68f8787 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sat, 5 Aug 2023 21:16:47 +0300 Subject: [PATCH] [#38] grafana: Add engine dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 743 ++++++++++++++++-- 1 file changed, 672 insertions(+), 71 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 6fd6bf5..0d08ded 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -403,7 +403,7 @@ "options": { "mode": "exclude", "names": [ - "neo.fs.v2.object.ObjectService.Search" + "neo.fs.v2.object.ObjectService.Put" ], "prefix": "All except:", "readOnly": true @@ -643,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 20 }, "id": 28, "options": { @@ -744,7 +744,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 30 }, "id": 29, "options": { @@ -844,7 +844,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 40 }, "id": 30, "options": { @@ -930,8 +930,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -947,7 +946,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 50 }, "id": 31, "options": { @@ -996,6 +995,634 @@ "x": 0, "y": 9 }, + "id": 58, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by storage engine service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_engine_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the engine requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_engine_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Payload size by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_payload_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Objects count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, type {{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Errors count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_errors_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Errors count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_engine_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Shard mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Engine", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, "id": 33, "panels": [ { @@ -1047,8 +1674,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1148,8 +1774,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1248,8 +1873,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1350,8 +1974,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1414,7 +2037,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 12, "panels": [ @@ -1467,8 +2090,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1568,8 +2190,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1668,8 +2289,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1768,8 +2388,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1868,8 +2487,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1968,8 +2586,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2070,8 +2687,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2134,7 +2750,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 38, "panels": [ @@ -2204,7 +2820,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 39, "options": { @@ -2305,7 +2921,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 40, "options": { @@ -2405,7 +3021,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 41, "options": { @@ -2508,7 +3124,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 43, "options": { @@ -2626,7 +3242,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "id": 42, "options": { @@ -2673,7 +3289,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 52, "panels": [ @@ -2726,8 +3342,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2827,8 +3442,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2927,8 +3541,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3030,8 +3643,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3148,8 +3760,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3212,7 +3823,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 44, "panels": [ @@ -3265,8 +3876,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3366,8 +3976,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3466,8 +4075,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3569,8 +4177,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3688,8 +4295,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3791,8 +4397,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3893,8 +4498,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3957,7 +4561,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 23, "panels": [ @@ -4007,8 +4611,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4107,8 +4710,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4207,8 +4809,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4271,7 +4872,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 6, "panels": [ @@ -4821,7 +5422,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" },