[#38] grafana: Add node instance state

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-08-02 16:14:29 +03:00
parent 4f9285251f
commit d5ee290740

View file

@ -23,93 +23,102 @@
"liveNow": false, "liveNow": false,
"panels": [ "panels": [
{ {
"collapsed": false, "datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Instance state",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": true,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [
{
"options": {
"0": {
"index": 0,
"text": "Undefined"
},
"1": {
"index": 1,
"text": "Starting"
},
"2": {
"index": 2,
"text": "Ready"
},
"3": {
"index": 3,
"text": "Shutting down"
}
},
"type": "value"
}
],
"max": 3,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "string"
},
"overrides": []
},
"gridPos": { "gridPos": {
"h": 1, "h": 6,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 0 "y": 0
}, },
"id": 2, "id": 21,
"panels": [],
"title": "Server",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Requests per second by gRPC method",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 100,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 1
},
"id": 1,
"options": { "options": {
"legend": { "legend": {
"calcs": [], "calcs": [],
"displayMode": "table", "displayMode": "table",
"placement": "right", "placement": "right",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "multi", "mode": "multi",
@ -125,217 +134,16 @@
}, },
"editorMode": "code", "editorMode": "code",
"exemplar": false, "exemplar": false,
"expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", "expr": "frostfs_node_state_health{job=\"node\", instance=\"$instance\"}\n",
"format": "time_series", "format": "time_series",
"instant": false, "instant": false,
"interval": "", "interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}", "legendFormat": "state",
"range": true, "range": true,
"refId": "A" "refId": "A"
} }
], ],
"title": "RPS", "title": "State",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Percentage of the failed requests",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 11
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "Error rate",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Latency of the requests for selected quantile",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "ms"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 21
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "Latency",
"transformations": [], "transformations": [],
"type": "timeseries" "type": "timeseries"
}, },
@ -345,7 +153,350 @@
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 31 "y": 6
},
"id": 2,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Requests per second by gRPC method",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 100,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 7
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "RPS",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Percentage of the failed requests",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"neo.fs.v2.object.ObjectService.Search"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 17
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "Error rate",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Latency of the requests for selected quantile",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "ms"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 27
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "Latency",
"transformations": [],
"type": "timeseries"
}
],
"title": "Server",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 7
}, },
"id": 6, "id": 6,
"panels": [ "panels": [
@ -398,8 +549,7 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green", "color": "green"
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -499,8 +649,7 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green", "color": "green"
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -599,8 +748,7 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green", "color": "green"
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -699,8 +847,7 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green", "color": "green"
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -833,8 +980,7 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green", "color": "green"
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -897,7 +1043,7 @@
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 32 "y": 8
}, },
"id": 12, "id": 12,
"panels": [ "panels": [
@ -967,7 +1113,7 @@
"h": 10, "h": 10,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 33 "y": 39
}, },
"id": 13, "id": 13,
"options": { "options": {
@ -1068,7 +1214,7 @@
"h": 10, "h": 10,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 43 "y": 49
}, },
"id": 15, "id": 15,
"options": { "options": {
@ -1168,7 +1314,7 @@
"h": 10, "h": 10,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 53 "y": 59
}, },
"id": 14, "id": 14,
"options": { "options": {
@ -1268,7 +1414,7 @@
"h": 10, "h": 10,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 63 "y": 69
}, },
"id": 16, "id": 16,
"options": { "options": {
@ -1368,7 +1514,7 @@
"h": 10, "h": 10,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 73 "y": 79
}, },
"id": 17, "id": 17,
"options": { "options": {
@ -1468,7 +1614,7 @@
"h": 10, "h": 10,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 83 "y": 89
}, },
"id": 18, "id": 18,
"options": { "options": {
@ -1570,7 +1716,7 @@
"h": 10, "h": 10,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 93 "y": 99
}, },
"id": 19, "id": 19,
"options": { "options": {