2023-07-28 15:40:52 +00:00
{
"annotations" : {
"list" : [
{
"builtIn" : 1 ,
"datasource" : {
"type" : "grafana" ,
"uid" : "-- Grafana --"
} ,
"enable" : true ,
"hide" : true ,
"iconColor" : "rgba(0, 211, 255, 1)" ,
"name" : "Annotations & Alerts" ,
"type" : "dashboard"
}
]
} ,
"description" : "" ,
"editable" : true ,
"fiscalYearStartMonth" : 0 ,
"graphTooltip" : 0 ,
"links" : [ ] ,
"liveNow" : false ,
"panels" : [
2023-08-03 12:31:19 +00:00
{
"collapsed" : false ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 0
} ,
"id" : 22 ,
"panels" : [ ] ,
2023-08-03 12:57:07 +00:00
"title" : "General" ,
2023-08-03 12:31:19 +00:00
"type" : "row"
} ,
2023-07-28 15:40:52 +00:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-02 13:14:29 +00:00
"description" : "Instance state" ,
2023-07-28 15:40:52 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2023-08-02 13:14:29 +00:00
"axisCenteredZero" : true ,
2023-07-28 15:40:52 +00:00
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
2023-08-02 13:14:29 +00:00
"fillOpacity" : 0 ,
2023-07-28 15:40:52 +00:00
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2023-08-02 13:14:29 +00:00
"lineInterpolation" : "linear" ,
2023-07-28 15:40:52 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
2023-08-02 13:14:29 +00:00
"mode" : "none"
2023-07-28 15:40:52 +00:00
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
2023-08-02 13:14:29 +00:00
"decimals" : 0 ,
"mappings" : [
{
"options" : {
"0" : {
"index" : 0 ,
"text" : "Undefined"
} ,
"1" : {
"index" : 1 ,
"text" : "Starting"
} ,
"2" : {
"index" : 2 ,
"text" : "Ready"
} ,
"3" : {
"index" : 3 ,
"text" : "Shutting down"
}
} ,
"type" : "value"
}
] ,
"max" : 3 ,
"min" : 0 ,
2023-07-28 15:40:52 +00:00
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-02 13:14:29 +00:00
"unit" : "string"
2023-07-28 15:40:52 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
2023-08-02 13:14:29 +00:00
"h" : 6 ,
2023-08-03 12:57:07 +00:00
"w" : 12 ,
2023-07-28 15:40:52 +00:00
"x" : 0 ,
2023-08-03 12:31:19 +00:00
"y" : 1
2023-07-28 15:40:52 +00:00
} ,
2023-08-02 13:14:29 +00:00
"id" : 21 ,
2023-07-28 15:40:52 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
2023-08-02 13:14:29 +00:00
"showLegend" : false
2023-07-28 15:40:52 +00:00
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-02 13:14:29 +00:00
"expr" : "frostfs_node_state_health{job=\"node\", instance=\"$instance\"}\n" ,
2023-07-28 15:40:52 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-02 13:14:29 +00:00
"legendFormat" : "state" ,
2023-07-28 15:40:52 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-02 13:14:29 +00:00
"title" : "State" ,
2023-07-28 15:40:52 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
2023-08-03 12:57:07 +00:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Current epoch on the instance" ,
"fieldConfig" : {
"defaults" : {
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 11:34:54 +00:00
"unit" : "short"
2023-08-03 12:57:07 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 6 ,
"w" : 12 ,
"x" : 12 ,
"y" : 1
} ,
"id" : 32 ,
"options" : {
2023-08-04 11:34:54 +00:00
"colorMode" : "none" ,
"graphMode" : "none" ,
"justifyMode" : "auto" ,
2023-08-03 12:57:07 +00:00
"orientation" : "auto" ,
"reduceOptions" : {
"calcs" : [
"lastNotNull"
] ,
"fields" : "" ,
"values" : false
} ,
2023-08-04 11:34:54 +00:00
"textMode" : "auto"
2023-08-03 12:57:07 +00:00
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"expr" : "frostfs_node_ir_epoch{job=\"node\", instance=\"$instance\"}" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Epoch" ,
2023-08-04 11:34:54 +00:00
"type" : "stat"
2023-08-03 12:57:07 +00:00
} ,
2023-07-28 15:40:52 +00:00
{
2023-08-04 10:59:12 +00:00
"collapsed" : true ,
2023-07-28 15:40:52 +00:00
"gridPos" : {
2023-08-02 13:14:29 +00:00
"h" : 1 ,
2023-07-28 15:40:52 +00:00
"w" : 24 ,
"x" : 0 ,
2023-08-03 12:31:19 +00:00
"y" : 7
2023-07-28 15:40:52 +00:00
} ,
2023-08-02 13:14:29 +00:00
"id" : 2 ,
2023-08-04 10:59:12 +00:00
"panels" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 10:59:12 +00:00
"description" : "Requests per second by gRPC method" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "reqps"
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 10:59:12 +00:00
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 8
} ,
"id" : 1 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 10:59:12 +00:00
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
2023-08-03 12:31:19 +00:00
}
} ,
2023-08-04 10:59:12 +00:00
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 10:59:12 +00:00
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{grpc_service}}.{{grpc_method}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "RPS" ,
"transformations" : [ ] ,
"type" : "timeseries"
2023-08-03 12:31:19 +00:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 10:59:12 +00:00
"description" : "Percentage of the failed requests" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 10:59:12 +00:00
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
2023-08-03 12:31:19 +00:00
"legend" : false ,
"tooltip" : false ,
2023-08-04 10:59:12 +00:00
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
2023-08-03 12:31:19 +00:00
}
2023-08-04 10:59:12 +00:00
} ,
"mappings" : [ ] ,
"max" : 100 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "percent"
} ,
"overrides" : [
{
"__systemRef" : "hideSeriesFrom" ,
"matcher" : {
"id" : "byNames" ,
"options" : {
"mode" : "exclude" ,
"names" : [
"neo.fs.v2.object.ObjectService.Search"
] ,
"prefix" : "All except:" ,
"readOnly" : true
}
} ,
"properties" : [
{
"id" : "custom.hideFrom" ,
"value" : {
"legend" : false ,
"tooltip" : false ,
"viz" : true
}
}
]
2023-08-03 12:31:19 +00:00
}
]
} ,
2023-08-04 10:59:12 +00:00
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 18
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 10:59:12 +00:00
"id" : 3 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 10:59:12 +00:00
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
2023-08-03 12:31:19 +00:00
}
} ,
2023-08-04 10:59:12 +00:00
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 10:59:12 +00:00
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{grpc_service}}.{{grpc_method}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Error rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
2023-08-03 12:31:19 +00:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 10:59:12 +00:00
"description" : "Latency of the requests for selected quantile" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "s"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 28
} ,
"id" : 4 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{grpc_service}}.{{grpc_method}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Latency" ,
"transformations" : [ ] ,
"type" : "timeseries"
2023-08-03 12:31:19 +00:00
}
] ,
2023-08-04 10:59:12 +00:00
"title" : "Server" ,
"type" : "row"
2023-08-03 12:31:19 +00:00
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 8
2023-08-03 12:31:19 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 27 ,
2023-08-02 13:14:29 +00:00
"panels" : [
2023-07-28 15:40:52 +00:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Requests per second by object service method" ,
2023-08-02 13:14:29 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green" ,
"value" : null
2023-08-02 13:14:29 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "reqps"
2023-07-28 15:40:52 +00:00
} ,
2023-08-02 13:14:29 +00:00
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 9
2023-08-02 13:14:29 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 28 ,
2023-08-02 13:14:29 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
2023-07-28 15:40:52 +00:00
} ,
2023-08-02 13:14:29 +00:00
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
2023-07-28 15:40:52 +00:00
}
} ,
2023-08-02 13:14:29 +00:00
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-02 13:14:29 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 13:11:35 +00:00
"legendFormat" : "{{method}}" ,
2023-08-02 13:14:29 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "RPS" ,
2023-08-02 13:14:29 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Percentage of the failed object service requests" ,
2023-08-02 13:14:29 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
2023-07-28 15:40:52 +00:00
} ,
2023-08-02 13:14:29 +00:00
"mappings" : [ ] ,
"max" : 100 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green" ,
"value" : null
2023-08-02 13:14:29 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "percent"
} ,
2023-08-03 12:31:19 +00:00
"overrides" : [ ]
2023-07-28 15:40:52 +00:00
} ,
2023-08-02 13:14:29 +00:00
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 19
2023-08-02 13:14:29 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 29 ,
2023-08-02 13:14:29 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n" ,
2023-08-02 13:14:29 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-02 13:14:29 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Error rate" ,
2023-08-02 13:14:29 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
2023-07-28 15:40:52 +00:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Latency of the object service requests for selected quantile" ,
2023-08-02 13:14:29 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green" ,
"value" : null
2023-08-02 13:14:29 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 10:59:12 +00:00
"unit" : "s"
2023-08-02 13:14:29 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 29
2023-08-02 13:14:29 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 30 ,
2023-08-02 13:14:29 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Latency" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Bytes per second by object service method" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 10:59:12 +00:00
"unit" : "binBps"
2023-08-03 13:11:35 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 39
2023-08-03 13:11:35 +00:00
} ,
"id" : 31 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Payload rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
}
] ,
"title" : "Object service" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 9
2023-08-03 13:11:35 +00:00
} ,
"id" : 33 ,
"panels" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Requests per second by metabase method for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "reqps"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 11:34:54 +00:00
"y" : 51
2023-08-03 13:11:35 +00:00
} ,
"id" : 35 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum by (instance, shard_id, method) (\n rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-02 13:14:29 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-02 13:14:29 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "RPS" ,
2023-08-02 13:14:29 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
2023-08-03 12:31:19 +00:00
} ,
2023-08-01 14:18:54 +00:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
2023-07-28 15:40:52 +00:00
} ,
2023-08-03 13:11:35 +00:00
"description" : "Percentage of the failed metabase requests for each shard" ,
2023-08-01 14:18:54 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
2023-08-03 12:31:19 +00:00
"fillOpacity" : 0 ,
2023-08-01 14:18:54 +00:00
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
2023-08-03 12:31:19 +00:00
"mode" : "none"
2023-08-01 14:18:54 +00:00
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
2023-08-03 13:11:35 +00:00
"max" : 100 ,
2023-08-01 14:18:54 +00:00
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "percent"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 11:34:54 +00:00
"y" : 61
2023-08-03 13:11:35 +00:00
} ,
"id" : 37 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Error rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Latency of the metabase requests for selected quantile for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
2023-08-01 14:18:54 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 10:59:12 +00:00
"unit" : "s"
2023-07-28 15:40:52 +00:00
} ,
2023-08-01 14:18:54 +00:00
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 11:34:54 +00:00
"y" : 71
2023-08-01 14:18:54 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 36 ,
2023-08-01 14:18:54 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
2023-07-28 15:40:52 +00:00
} ,
2023-08-03 12:31:19 +00:00
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_metabase_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n" ,
2023-08-03 12:31:19 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 13:11:35 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-03 12:31:19 +00:00
"range" : true ,
"refId" : "A"
2023-08-01 14:18:54 +00:00
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Latency" ,
2023-08-01 14:18:54 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Metabase mode for each shard" ,
2023-08-01 14:18:54 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2023-08-03 12:31:19 +00:00
"lineInterpolation" : "linear" ,
2023-08-01 14:18:54 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
2023-08-03 12:31:19 +00:00
"max" : 1.5 ,
"min" : 0 ,
2023-08-01 14:18:54 +00:00
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green" ,
"value" : null
2023-08-01 14:18:54 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-03 12:31:19 +00:00
"unit" : "none"
2023-07-28 15:40:52 +00:00
} ,
2023-08-01 14:18:54 +00:00
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 11:34:54 +00:00
"y" : 81
2023-08-01 14:18:54 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 34 ,
2023-08-01 14:18:54 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
2023-07-28 15:40:52 +00:00
}
} ,
2023-08-01 14:18:54 +00:00
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
2023-07-28 15:40:52 +00:00
} ,
2023-08-01 14:18:54 +00:00
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "sum(frostfs_node_metabase_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n" ,
2023-08-01 14:18:54 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "Shard {{shard_id}}, mode {{mode}}" ,
2023-08-01 14:18:54 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Mode" ,
2023-08-01 14:18:54 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
2023-08-03 12:31:19 +00:00
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Metabase" ,
2023-08-03 12:31:19 +00:00
"type" : "row"
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 10
2023-08-03 12:31:19 +00:00
} ,
"id" : 12 ,
"panels" : [
2023-08-01 14:18:54 +00:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
2023-07-28 15:40:52 +00:00
} ,
2023-08-03 12:31:19 +00:00
"description" : "Requests per second by writecache method for each shard" ,
2023-08-01 14:18:54 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
2023-08-03 12:31:19 +00:00
"fillOpacity" : 100 ,
2023-08-01 14:18:54 +00:00
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
2023-08-03 12:31:19 +00:00
"lineStyle" : {
"fill" : "solid"
} ,
2023-08-01 14:18:54 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
2023-08-03 12:31:19 +00:00
"mode" : "normal"
2023-08-01 14:18:54 +00:00
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 12:31:19 +00:00
"color" : "green" ,
"value" : null
2023-08-01 14:18:54 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-03 12:31:19 +00:00
"unit" : "reqps"
2023-08-01 14:18:54 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 11
2023-08-01 14:18:54 +00:00
} ,
2023-08-03 12:31:19 +00:00
"id" : 13 ,
2023-08-01 14:18:54 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 12:31:19 +00:00
"expr" : "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-01 14:18:54 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 12:31:19 +00:00
"title" : "RPS" ,
2023-08-01 14:18:54 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
2023-07-28 15:40:52 +00:00
} ,
2023-08-01 14:18:54 +00:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 12:31:19 +00:00
"description" : "Percentage of the failed writecache requests for each shard" ,
2023-08-01 14:18:54 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
2023-08-03 12:31:19 +00:00
"max" : 100 ,
2023-08-01 14:18:54 +00:00
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 12:31:19 +00:00
"color" : "green" ,
"value" : null
2023-08-01 14:18:54 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-03 12:31:19 +00:00
"unit" : "percent"
2023-08-01 14:18:54 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 21
2023-08-01 14:18:54 +00:00
} ,
2023-08-03 12:31:19 +00:00
"id" : 15 ,
2023-08-01 14:18:54 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 12:31:19 +00:00
"expr" : "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n" ,
2023-08-01 14:18:54 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-01 14:18:54 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 12:31:19 +00:00
"title" : "Error rate" ,
2023-08-01 14:18:54 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
2023-07-28 15:40:52 +00:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Latency of the writecache requests for selected quantile for each shard" ,
2023-08-01 14:18:54 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2023-08-03 12:31:19 +00:00
"lineInterpolation" : "smooth" ,
2023-08-01 14:18:54 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 12:31:19 +00:00
"color" : "green" ,
"value" : null
2023-08-01 14:18:54 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 10:59:12 +00:00
"unit" : "s"
2023-08-01 14:18:54 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 31
2023-08-01 14:18:54 +00:00
} ,
2023-08-03 12:31:19 +00:00
"id" : 14 ,
2023-08-01 14:18:54 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 12:31:19 +00:00
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n" ,
2023-08-01 14:18:54 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-01 14:18:54 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 12:31:19 +00:00
"title" : "Latency" ,
2023-08-01 14:18:54 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
2023-08-03 12:31:19 +00:00
} ,
2023-08-02 12:06:03 +00:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 12:31:19 +00:00
"description" : "Actual count of the objects, that writecache stores" ,
2023-08-02 12:06:03 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
2023-08-03 12:31:19 +00:00
"fillOpacity" : 0 ,
2023-08-02 12:06:03 +00:00
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
2023-08-03 12:31:19 +00:00
"mode" : "none"
2023-08-02 12:06:03 +00:00
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-03 12:31:19 +00:00
"unit" : "none"
2023-08-02 12:06:03 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 41
2023-08-02 12:06:03 +00:00
} ,
2023-08-03 12:31:19 +00:00
"id" : 16 ,
2023-08-02 12:06:03 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 12:31:19 +00:00
"expr" : "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}" ,
2023-08-02 12:06:03 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "Shard {{shard_id}}, storage {{storage}}" ,
2023-08-02 12:06:03 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 12:31:19 +00:00
"title" : "Actual count" ,
2023-08-02 12:06:03 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 12:31:19 +00:00
"description" : "Estimated size of the objects, that writecache stores" ,
2023-08-02 12:06:03 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 10:59:12 +00:00
"unit" : "bytes"
2023-08-02 12:06:03 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 51
2023-08-02 12:06:03 +00:00
} ,
2023-08-03 12:31:19 +00:00
"id" : 17 ,
2023-08-02 12:06:03 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 12:31:19 +00:00
"expr" : "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}" ,
2023-08-02 12:06:03 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "Shard {{shard_id}}, storage {{storage}}" ,
2023-08-02 12:06:03 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 12:31:19 +00:00
"title" : "Estimated size" ,
2023-08-02 12:06:03 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 12:31:19 +00:00
"description" : "Rate of the writecache background operations" ,
2023-08-02 12:06:03 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-03 12:31:19 +00:00
"unit" : "none"
2023-08-02 12:06:03 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 61
2023-08-02 12:06:03 +00:00
} ,
2023-08-03 12:31:19 +00:00
"id" : 18 ,
2023-08-02 12:06:03 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 12:31:19 +00:00
"expr" : "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))" ,
2023-08-02 12:06:03 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}" ,
2023-08-02 12:06:03 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 12:31:19 +00:00
"title" : "Operations rate" ,
2023-08-02 12:06:03 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 12:31:19 +00:00
"description" : "Writecache mode for each shard" ,
2023-08-02 12:06:03 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2023-08-03 12:31:19 +00:00
"lineInterpolation" : "linear" ,
2023-08-02 12:06:03 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
2023-08-03 12:31:19 +00:00
"max" : 1.5 ,
"min" : 0 ,
2023-08-02 12:06:03 +00:00
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "none"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 71
2023-08-02 12:06:03 +00:00
} ,
2023-08-03 12:31:19 +00:00
"id" : 19 ,
2023-08-02 12:06:03 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 12:31:19 +00:00
"expr" : "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n" ,
2023-08-02 12:06:03 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 12:31:19 +00:00
"legendFormat" : "Shard {{shard_id}}, mode {{mode}}" ,
2023-08-02 12:06:03 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 12:31:19 +00:00
"title" : "Mode" ,
2023-08-02 12:06:03 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
2023-08-03 12:31:19 +00:00
}
] ,
"title" : "Writecache" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 10:59:12 +00:00
"y" : 11
2023-08-03 12:31:19 +00:00
} ,
2023-08-04 11:34:54 +00:00
"id" : 38 ,
2023-08-03 12:31:19 +00:00
"panels" : [
2023-08-02 12:06:03 +00:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 11:34:54 +00:00
"description" : "Requests per second by blobstore method for each shard" ,
2023-08-02 12:06:03 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
2023-08-04 11:34:54 +00:00
"fillOpacity" : 100 ,
2023-08-02 12:06:03 +00:00
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
2023-08-04 11:34:54 +00:00
"lineStyle" : {
"fill" : "solid"
} ,
2023-08-02 12:06:03 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
2023-08-04 11:34:54 +00:00
"mode" : "normal"
2023-08-02 12:06:03 +00:00
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 11:34:54 +00:00
"unit" : "reqps"
2023-08-02 12:06:03 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 12
} ,
"id" : 39 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "RPS" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Percentage of the failed blobstore requests for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"max" : 100 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "percent"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 22
} ,
"id" : 40 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Error rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Latency of the metabase requests for selected quantile for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "s"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 32
} ,
"id" : 41 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobstore_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Latency" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Bytes per second by blobstore shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "binBps"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 42
} ,
"id" : 43 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
2023-08-04 11:50:04 +00:00
} ,
2023-08-04 12:03:48 +00:00
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Write rate, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
"format" : "time_series" ,
"hide" : false ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Read rate, shard {{shard_id}}" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Payload rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Blobstore mode for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"max" : 1.5 ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "none"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 52
} ,
"id" : 42 ,
2023-08-04 11:50:04 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 12:03:48 +00:00
"expr" : "sum(frostfs_node_blobstore_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Shard {{shard_id}}, mode {{mode}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Mode" ,
"transformations" : [ ] ,
"type" : "timeseries"
}
] ,
"title" : "Blobstore" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 12
} ,
"id" : 52 ,
"panels" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Requests per second by fstree method for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "reqps"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 63
} ,
"id" : 53 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum by (instance, shard_id, method) (\n rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-04 11:50:04 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "RPS" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 12:03:48 +00:00
"description" : "Percentage of the failed fstree requests for each shard" ,
2023-08-04 11:50:04 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"max" : 100 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "percent"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 73
2023-08-04 11:50:04 +00:00
} ,
2023-08-04 12:03:48 +00:00
"id" : 54 ,
2023-08-04 11:50:04 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 12:03:48 +00:00
"expr" : "sum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n" ,
2023-08-04 11:50:04 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Error rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 12:03:48 +00:00
"description" : "Latency of the fstree requests for selected quantile for each shard" ,
2023-08-04 11:50:04 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "s"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 83
2023-08-04 11:50:04 +00:00
} ,
2023-08-04 12:03:48 +00:00
"id" : 55 ,
2023-08-04 11:50:04 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 12:03:48 +00:00
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_fstree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n" ,
2023-08-04 11:50:04 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Latency" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 12:03:48 +00:00
"description" : "Bytes per second by fstree shard" ,
2023-08-04 11:50:04 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "binBps"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 93
2023-08-04 11:50:04 +00:00
} ,
2023-08-04 12:03:48 +00:00
"id" : 56 ,
2023-08-04 11:50:04 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 12:03:48 +00:00
"expr" : "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-04 11:50:04 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Write rate, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 12:03:48 +00:00
"expr" : "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-04 11:50:04 +00:00
"format" : "time_series" ,
"hide" : false ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Read rate, shard {{shard_id}}" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Payload rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 12:03:48 +00:00
"description" : "FSTree mode for each shard" ,
2023-08-04 11:50:04 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"max" : 1.5 ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "none"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 103
2023-08-04 11:50:04 +00:00
} ,
2023-08-04 12:03:48 +00:00
"id" : 57 ,
2023-08-04 11:50:04 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 12:03:48 +00:00
"expr" : "sum(frostfs_node_fstree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n" ,
2023-08-04 11:50:04 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Shard {{shard_id}}, mode {{mode}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Mode" ,
"transformations" : [ ] ,
"type" : "timeseries"
}
] ,
2023-08-04 12:03:48 +00:00
"title" : "FSTree" ,
2023-08-04 11:50:04 +00:00
"type" : "row"
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 13
2023-08-04 11:50:04 +00:00
} ,
"id" : 44 ,
"panels" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Requests per second by blobovnizca method for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "reqps"
} ,
"overrides" : [ ]
2023-08-02 12:06:03 +00:00
} ,
2023-08-04 11:50:04 +00:00
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 64
2023-08-04 11:50:04 +00:00
} ,
"id" : 45 ,
2023-08-02 12:06:03 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 11:50:04 +00:00
"expr" : "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-02 12:06:03 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-04 11:34:54 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-02 12:06:03 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-04 11:34:54 +00:00
"title" : "RPS" ,
2023-08-02 12:06:03 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 11:50:04 +00:00
"description" : "Percentage of the failed blobovnizca requests for each shard" ,
2023-08-02 12:06:03 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
2023-08-04 11:34:54 +00:00
"max" : 100 ,
2023-08-02 12:06:03 +00:00
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 11:34:54 +00:00
"unit" : "percent"
2023-08-02 12:06:03 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 74
2023-08-02 12:06:03 +00:00
} ,
2023-08-04 11:50:04 +00:00
"id" : 46 ,
2023-08-02 12:06:03 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 11:50:04 +00:00
"expr" : "sum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n" ,
2023-08-02 12:06:03 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-04 11:34:54 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-02 12:06:03 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-04 11:34:54 +00:00
"title" : "Error rate" ,
2023-08-02 12:06:03 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 11:50:04 +00:00
"description" : "Latency of the blobovnizca requests for selected quantile for each shard" ,
2023-08-02 12:06:03 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2023-08-03 12:31:19 +00:00
"lineInterpolation" : "smooth" ,
2023-08-02 12:06:03 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-04 11:34:54 +00:00
"unit" : "s"
2023-08-02 12:06:03 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 84
2023-08-02 12:06:03 +00:00
} ,
2023-08-04 11:50:04 +00:00
"id" : 47 ,
2023-08-02 12:06:03 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 11:50:04 +00:00
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovniza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n" ,
2023-08-02 12:06:03 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-04 11:34:54 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-02 12:06:03 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-04 11:34:54 +00:00
"title" : "Latency" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 11:50:04 +00:00
"description" : "Bytes per second by blobovnizca shard" ,
2023-08-04 11:34:54 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "binBps"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 94
2023-08-04 11:34:54 +00:00
} ,
2023-08-04 11:50:04 +00:00
"id" : 48 ,
2023-08-04 11:34:54 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 11:50:04 +00:00
"expr" : "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-04 11:34:54 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Write rate, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 11:50:04 +00:00
"expr" : "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-04 11:34:54 +00:00
"format" : "time_series" ,
"hide" : false ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Read rate, shard {{shard_id}}" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Payload rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-04 11:50:04 +00:00
"description" : "Blobovnizca size for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "bytes"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 104
2023-08-04 11:50:04 +00:00
} ,
"id" : 50 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "frostfs_node_blobovniza_tree_size_bytes{instance=\"$instance\"}\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Size" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Count of open DB's of blobovnizca for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "none"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 114
2023-08-04 11:50:04 +00:00
} ,
"id" : 51 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "frostfs_node_blobovniza_tree_open_blobovnizca_count{instance=\"$instance\"}\n" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Open DB count" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Blobovnizca mode for each shard" ,
2023-08-04 11:34:54 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"max" : 1.5 ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "none"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 124
2023-08-04 11:34:54 +00:00
} ,
2023-08-04 11:50:04 +00:00
"id" : 49 ,
2023-08-04 11:34:54 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-04 11:50:04 +00:00
"expr" : "sum(frostfs_node_blobovniza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n" ,
2023-08-04 11:34:54 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "Shard {{shard_id}}, mode {{mode}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Mode" ,
2023-08-02 12:06:03 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
}
] ,
2023-08-04 11:50:04 +00:00
"title" : "Blobovnizca" ,
2023-08-02 12:06:03 +00:00
"type" : "row"
2023-08-03 12:48:53 +00:00
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 14
2023-08-03 12:48:53 +00:00
} ,
2023-08-04 11:34:54 +00:00
"id" : 23 ,
"panels" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Count of in-flight requests" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "none"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 12
} ,
"id" : 24 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "in-flight requests" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "In-flight request" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Rate of the objects replicated" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "none"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 22
} ,
"id" : 25 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "object processing rate" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Objects processing rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Rate of the objects payload replicated" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "binBps"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 32
} ,
"id" : 26 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "payload processing rate" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Payload processing rate" ,
"transformations" : [ ] ,
"type" : "timeseries"
}
] ,
"title" : "Replicator" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2023-08-04 12:03:48 +00:00
"y" : 15
2023-08-04 11:34:54 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 6 ,
2023-08-03 12:48:53 +00:00
"panels" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Requests per second by pilorama method for each shard" ,
2023-08-03 12:48:53 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineStyle" : {
"fill" : "solid"
} ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green"
2023-08-03 12:48:53 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "reqps"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-03 13:11:35 +00:00
"y" : 40
2023-08-03 12:48:53 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 8 ,
2023-08-03 12:48:53 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "sum by (instance, shard_id, method) (\n rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n" ,
2023-08-03 12:48:53 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 13:11:35 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-03 12:48:53 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Pilorama RPS" ,
2023-08-03 12:48:53 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Percentage of the failed pilorama requests for each shard" ,
2023-08-03 12:48:53 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"max" : 100 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green"
2023-08-03 12:48:53 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "percent"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-03 13:11:35 +00:00
"y" : 50
2023-08-03 12:48:53 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 9 ,
2023-08-03 12:48:53 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "sum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n" ,
2023-08-03 12:48:53 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Pilorama error rate" ,
2023-08-03 12:48:53 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Latency of the pilorama requests for selected quantile for each shard" ,
2023-08-03 12:48:53 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green"
2023-08-03 12:48:53 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-03 13:11:35 +00:00
"y" : 60
2023-08-03 12:48:53 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 10 ,
2023-08-03 12:48:53 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method, le)\n)\n" ,
2023-08-03 12:48:53 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 13:11:35 +00:00
"legendFormat" : "{{method}}, shard {{shard_id}}" ,
2023-08-03 12:48:53 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Pilorama latency" ,
2023-08-03 12:48:53 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
2023-08-03 13:11:35 +00:00
"description" : "Duration of waiting time for replication, replication task duration and synchronization for selected quantile" ,
2023-08-03 12:48:53 +00:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
2023-08-03 13:11:35 +00:00
"fillOpacity" : 0 ,
2023-08-03 12:48:53 +00:00
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "smooth" ,
2023-08-03 13:11:35 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green"
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
"y" : 70
} ,
"id" : 7 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_replicate_wait_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "replicate wait success:{{success}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "histogram_quantile($quantile,\n sum(rate(frostfs_node_treeservice_replicate_task_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)" ,
"format" : "time_series" ,
"hide" : false ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "replicate task success:{{success}}" ,
"range" : true ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_sync_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)" ,
"format" : "time_series" ,
"hide" : false ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "sync success:{{success}}" ,
"range" : true ,
"refId" : "C"
}
] ,
"title" : "Replication and synchronization" ,
"transformations" : [ ] ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"description" : "Pilorama mode for each shard" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
2023-08-03 12:48:53 +00:00
} ,
2023-08-03 13:11:35 +00:00
"lineInterpolation" : "linear" ,
2023-08-03 12:48:53 +00:00
"lineWidth" : 1 ,
"pointSize" : 3 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
2023-08-03 13:11:35 +00:00
"mode" : "none"
2023-08-03 12:48:53 +00:00
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
2023-08-03 13:11:35 +00:00
"max" : 1.5 ,
"min" : 0 ,
2023-08-03 12:48:53 +00:00
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-08-03 13:11:35 +00:00
"color" : "green"
2023-08-03 12:48:53 +00:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2023-08-03 13:11:35 +00:00
"unit" : "none"
2023-08-03 12:48:53 +00:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 10 ,
"w" : 24 ,
"x" : 0 ,
2023-08-03 13:11:35 +00:00
"y" : 80
2023-08-03 12:48:53 +00:00
} ,
2023-08-03 13:11:35 +00:00
"id" : 11 ,
2023-08-03 12:48:53 +00:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.5.6" ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2023-08-03 13:11:35 +00:00
"expr" : "sum(frostfs_node_pilorama_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n" ,
2023-08-03 12:48:53 +00:00
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
2023-08-03 13:11:35 +00:00
"legendFormat" : "Shard {{shard_id}}, mode {{mode}}" ,
2023-08-03 12:48:53 +00:00
"range" : true ,
"refId" : "A"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Pilorama mode" ,
2023-08-03 12:48:53 +00:00
"transformations" : [ ] ,
"type" : "timeseries"
}
] ,
2023-08-03 13:11:35 +00:00
"title" : "Tree service" ,
2023-08-03 12:48:53 +00:00
"type" : "row"
2023-07-28 15:40:52 +00:00
}
] ,
"refresh" : "1m" ,
"schemaVersion" : 38 ,
"style" : "dark" ,
"tags" : [ ] ,
"templating" : {
"list" : [
{
"current" : {
2023-08-04 11:34:54 +00:00
"selected" : true ,
2023-07-28 15:40:52 +00:00
"text" : "s01.frostfs.devenv:9090" ,
"value" : "s01.frostfs.devenv:9090"
} ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "PBFA97CFB590B2093"
} ,
"definition" : "label_values(frostfs_node_state_health,instance)" ,
"hide" : 0 ,
"includeAll" : false ,
2023-08-01 14:18:54 +00:00
"label" : "Instance" ,
2023-07-28 15:40:52 +00:00
"multi" : false ,
"name" : "instance" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(frostfs_node_state_health,instance)" ,
"refId" : "PrometheusVariableQueryEditor-VariableQuery"
} ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"type" : "query"
2023-08-01 14:18:54 +00:00
} ,
{
"current" : {
"selected" : false ,
"text" : "0.95" ,
"value" : "0.95"
} ,
"hide" : 0 ,
"includeAll" : false ,
"label" : "Quantile" ,
"multi" : false ,
"name" : "quantile" ,
"options" : [
{
"selected" : true ,
"text" : "0.95" ,
"value" : "0.95"
} ,
{
"selected" : false ,
"text" : "0.99" ,
"value" : "0.99"
} ,
{
"selected" : false ,
"text" : "1.00" ,
"value" : "1.00"
}
] ,
"query" : "0.95, 0.99, 1.00" ,
"queryValue" : "" ,
"skipUrlSync" : false ,
"type" : "custom"
2023-07-28 15:40:52 +00:00
}
]
} ,
"time" : {
2023-08-01 14:18:54 +00:00
"from" : "now-1h" ,
2023-07-28 15:40:52 +00:00
"to" : "now"
} ,
"timepicker" : { } ,
"timezone" : "" ,
"title" : "Storage node" ,
"uid" : "aa6318b6-4780-4a9a-9fe7-a4a81917265f" ,
"version" : 1 ,
"weekStart" : ""
}
