[#38] grafana: Add node instance state

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
master
Dmitrii Stepanov 2023-08-02 16:14:29 +03:00
parent 4f9285251f
commit d5ee290740
1 changed files with 446 additions and 300 deletions

View File

@ -23,93 +23,102 @@
"liveNow": false,
"panels": [
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Instance state",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": true,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [
{
"options": {
"0": {
"index": 0,
"text": "Undefined"
},
"1": {
"index": 1,
"text": "Starting"
},
"2": {
"index": 2,
"text": "Ready"
},
"3": {
"index": 3,
"text": "Shutting down"
}
},
"type": "value"
}
],
"max": 3,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "string"
},
"overrides": []
},
"gridPos": {
"h": 1,
"h": 6,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"panels": [],
"title": "Server",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Requests per second by gRPC method",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 100,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 1
},
"id": 1,
"id": 21,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "multi",
@ -125,217 +134,16 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n",
"expr": "frostfs_node_state_health{job=\"node\", instance=\"$instance\"}\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"legendFormat": "state",
"range": true,
"refId": "A"
}
],
"title": "RPS",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Percentage of the failed requests",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 11
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "Error rate",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Latency of the requests for selected quantile",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "ms"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 21
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "Latency",
"title": "State",
"transformations": [],
"type": "timeseries"
},
@ -345,7 +153,350 @@
"h": 1,
"w": 24,
"x": 0,
"y": 31
"y": 6
},
"id": 2,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Requests per second by gRPC method",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 100,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 7
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "RPS",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Percentage of the failed requests",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"neo.fs.v2.object.ObjectService.Search"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 17
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "Error rate",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Latency of the requests for selected quantile",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 3,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "ms"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 27
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.5.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n",
"format": "time_series",
"instant": false,
"interval": "",
"legendFormat": "{{grpc_service}}.{{grpc_method}}",
"range": true,
"refId": "A"
}
],
"title": "Latency",
"transformations": [],
"type": "timeseries"
}
],
"title": "Server",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 7
},
"id": 6,
"panels": [
@ -398,8 +549,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -499,8 +649,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -599,8 +748,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -699,8 +847,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -833,8 +980,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -897,7 +1043,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 32
"y": 8
},
"id": 12,
"panels": [
@ -967,7 +1113,7 @@
"h": 10,
"w": 24,
"x": 0,
"y": 33
"y": 39
},
"id": 13,
"options": {
@ -1068,7 +1214,7 @@
"h": 10,
"w": 24,
"x": 0,
"y": 43
"y": 49
},
"id": 15,
"options": {
@ -1168,7 +1314,7 @@
"h": 10,
"w": 24,
"x": 0,
"y": 53
"y": 59
},
"id": 14,
"options": {
@ -1268,7 +1414,7 @@
"h": 10,
"w": 24,
"x": 0,
"y": 63
"y": 69
},
"id": 16,
"options": {
@ -1368,7 +1514,7 @@
"h": 10,
"w": 24,
"x": 0,
"y": 73
"y": 79
},
"id": 17,
"options": {
@ -1468,7 +1614,7 @@
"h": 10,
"w": 24,
"x": 0,
"y": 83
"y": 89
},
"id": 18,
"options": {
@ -1570,7 +1716,7 @@
"h": 10,
"w": 24,
"x": 0,
"y": 93
"y": 99
},
"id": 19,
"options": {