From 3400bb5736f769f5cf9720a3dbfff99809337c7a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 13 Jul 2023 12:13:27 +0300 Subject: [PATCH 01/81] [#37] Add grafana service to visualize prometheus metrics Signed-off-by: Alex Vanin --- .env | 4 + .services | 1 + services/grafana/.env | 1 + services/grafana/.hosts | 1 + services/grafana/.int_test.env | 1 + services/grafana/docker-compose.yml | 26 + services/grafana/grafana.ini | 3 + .../provisioning/dashboards/14061_rev1.json | 1162 +++++++++++++++++ .../provisioning/dashboards/dashboard.yml | 10 + .../provisioning/datasources/datasource.yml | 8 + 10 files changed, 1217 insertions(+) create mode 120000 services/grafana/.env create mode 100644 services/grafana/.hosts create mode 120000 services/grafana/.int_test.env create mode 100644 services/grafana/docker-compose.yml create mode 100644 services/grafana/grafana.ini create mode 100644 services/grafana/provisioning/dashboards/14061_rev1.json create mode 100644 services/grafana/provisioning/dashboards/dashboard.yml create mode 100644 services/grafana/provisioning/datasources/datasource.yml diff --git a/.env b/.env index f921d46..e04d5ca 100644 --- a/.env +++ b/.env @@ -60,3 +60,7 @@ JAEGER_IMAGE=jaegertracing/all-in-one # Prometheus monitoring PROMETHEUS_VERSION=v2.43.0 PROMETHEUS_IMAGE=prom/prometheus + +# Grafana versions +GRAFANA_VERSION=9.5.6 +GRAFANA_IMAGE=grafana/grafana diff --git a/.services b/.services index d9f34c7..768e4e6 100644 --- a/.services +++ b/.services @@ -3,3 +3,4 @@ http_gate s3_gate rest_gate +grafana diff --git a/services/grafana/.env b/services/grafana/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/grafana/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/grafana/.hosts b/services/grafana/.hosts new file mode 100644 index 0000000..8f36ea6 --- /dev/null +++ b/services/grafana/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.122 grafana.LOCAL_DOMAIN diff --git a/services/grafana/.int_test.env b/services/grafana/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/grafana/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml new file mode 100644 index 0000000..795b87c --- /dev/null +++ b/services/grafana/docker-compose.yml @@ -0,0 +1,26 @@ +version: '2.4' +services: + grafana: + image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: grafana + container_name: grafana + restart: on-failure + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.122 + volumes: + - ./../../vendor/hosts:/etc/hosts + - ./grafana.ini:/etc/grafana/grafana.ini + - ./provisioning:/etc/grafana/provisioning + ports: + - '3000:3000' + stop_signal: SIGKILL + env_file: [ ".env", ".int_test.env" ] + +networks: + grafana_int: + internet: + external: true + name: basenet_internet diff --git a/services/grafana/grafana.ini b/services/grafana/grafana.ini new file mode 100644 index 0000000..256888f --- /dev/null +++ b/services/grafana/grafana.ini @@ -0,0 +1,3 @@ +[auth.anonymous] +# enable anonymous access +enabled = true diff --git a/services/grafana/provisioning/dashboards/14061_rev1.json b/services/grafana/provisioning/dashboards/14061_rev1.json new file mode 100644 index 0000000..deec62b --- /dev/null +++ b/services/grafana/provisioning/dashboards/14061_rev1.json @@ -0,0 +1,1162 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A quickstart to setup the Prometheus Go runtime exporter with preconfigured dashboards, alerting rules, and recording rules.", + "editable": true, + "graphTooltip": 0, + "id": null, + "iteration": 1602794777869, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average total bytes of memory reserved across all process instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by(job)(go_memstats_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}} (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Reserved Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average stack memory usage across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job) (go_memstats_stack_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: stack inuse (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Stack Memory Use", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average memory reservations by the runtime, not for stack or heap, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_mspan_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: mspan (avg)", + "refId": "B" + }, + { + "expr": "avg by (job)(go_memstats_mcache_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: mcache (avg)", + "refId": "D" + }, + { + "expr": "avg by (job)(go_memstats_buck_hash_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: buck hash (avg)", + "refId": "E" + }, + { + "expr": "avg by (job)(go_memstats_gc_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: gc (avg)", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Other Memory Reservations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average memory reserved, and actually in use, by the heap, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_heap_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: heap reserved (avg)", + "refId": "B" + }, + { + "expr": "avg by (job)(go_memstats_heap_inuse_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: heap in use (avg)", + "refId": "A" + }, + { + "expr": "avg by (job)(go_memstats_heap_alloc_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap alloc (avg)", + "refId": "C" + }, + { + "expr": "avg by (job)(go_memstats_heap_idle_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap idle (avg)", + "refId": "D" + }, + { + "expr": "avg by (job)(go_memstats_heap_released_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap released (avg)", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average allocation rate in bytes per second, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(rate(go_memstats_alloc_bytes_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{job}}: bytes malloced/s (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocation Rate, Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average rate of heap object allocation, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_mallocs_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{job}}: obj mallocs/s (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Object Allocation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average number of live memory objects across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by(job)(go_memstats_mallocs_total{job=\"$job\", instance=~\"$instance\"} - go_memstats_frees_total{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: object count (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Live Objects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average number of goroutines across instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_goroutines{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: goroutine count (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_gc_duration_seconds{quantile=\"0\", job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: min gc time (avg)", + "refId": "A" + }, + { + "expr": "avg by (job)(go_gc_duration_seconds{quantile=\"1\", job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: max gc time (avg)", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC min & max duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The number used bytes at which the runtime plans to perform the next GC, averaged across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_next_gc_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}} next gc bytes (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Next GC, Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 25, + "style": "dark", + "tags": [ + "go", + "golang" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "definition": "label_values(go_info, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(go_info, job)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(go_info{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(go_info{job=\"$job\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Go Runtime Exporter Quickstart and Dashboardby Grafana Labs", + "uid": "CgCw8jKZz3", + "version": 3, + "gnetId": 14061 +} \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/dashboard.yml b/services/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..bc9f26e --- /dev/null +++ b/services/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: Prometheus + orgId: 1 + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..81b6a7b --- /dev/null +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 From 2744f675aa6efb52b046e9ee9a57d6fd9d9f0a1a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Jul 2023 18:40:52 +0300 Subject: [PATCH 02/81] [#38] grafana: Add storage dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 489 ++++++++++++++++++ 1 file changed, 489 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/storage-node.json diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json new file mode 100644 index 0000000..8f38c0d --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -0,0 +1,489 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "95%-tile latency of the requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency, 0.95q", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "99%-tile latency of the requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency, 0.99q", + "transformations": [], + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_state_health,instance)", + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_state_health,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node", + "uid": "aa6318b6-4780-4a9a-9fe7-a4a81917265f", + "version": 1, + "weekStart": "" +} \ No newline at end of file From 584fa43ca7c0617d97eb874b4e38a806ad8a9e37 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:05:40 +0300 Subject: [PATCH 03/81] [#38] grafana: Add overview board Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/overview.json | 327 ++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/overview.json diff --git a/services/grafana/provisioning/dashboards/overview.json b/services/grafana/provisioning/dashboards/overview.json new file mode 100644 index 0000000..46a5987 --- /dev/null +++ b/services/grafana/provisioning/dashboards/overview.json @@ -0,0 +1,327 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (\n rate(grpc_server_started_total[$__rate_interval])\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Error rate by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$__rate_interval])) by (instance)\n / \nsum(rate(grpc_server_started_total{}[$__rate_interval])) by (instance)\n * 100.0", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Resident memory size by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (\n process_resident_memory_bytes\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "da5f53f7-c069-44b5-8372-462333c9fe4a", + "version": 1, + "weekStart": "" + } \ No newline at end of file From 3abb217d3028937a4f50c506b7a16137dec9d4c6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:06:15 +0300 Subject: [PATCH 04/81] [#38] grafana: Change start order Start grafana right after prometheus. Signed-off-by: Dmitrii Stepanov --- .bootstrap_services | 1 + .services | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.bootstrap_services b/.bootstrap_services index 02237d1..a56729d 100644 --- a/.bootstrap_services +++ b/.bootstrap_services @@ -4,3 +4,4 @@ basenet morph_chain jaeger prometheus +grafana diff --git a/.services b/.services index 768e4e6..d9f34c7 100644 --- a/.services +++ b/.services @@ -3,4 +3,3 @@ http_gate s3_gate rest_gate -grafana From 7de23fe789d7817d553287d10ba312e2f867c2df Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:06:51 +0300 Subject: [PATCH 05/81] [#38] grafana: Fix config Allow to acces grafana anonymous. Change home page. Signed-off-by: Dmitrii Stepanov --- services/grafana/grafana.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/grafana/grafana.ini b/services/grafana/grafana.ini index 256888f..3c14906 100644 --- a/services/grafana/grafana.ini +++ b/services/grafana/grafana.ini @@ -1,3 +1,7 @@ [auth.anonymous] -# enable anonymous access enabled = true +org_name = Main Org. +org_role = Editor + +[dashboards] +default_home_dashboard_path= /etc/grafana/provisioning/dashboards/overview.json \ No newline at end of file From 2efc0442f18120cc05ea8fc287a70f284b4a6a37 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Aug 2023 16:14:18 +0300 Subject: [PATCH 06/81] [#38] grafana: Add instance to panels Signed-off-by: Dmitrii Stepanov --- services/grafana/provisioning/dashboards/storage-node.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 8f38c0d..62de402 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -19,7 +19,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, "links": [], "liveNow": false, "panels": [ @@ -327,7 +326,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -427,7 +426,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", From f749581c4ebd4e43932965fdf93f16b070dc2380 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Aug 2023 17:18:54 +0300 Subject: [PATCH 07/81] [#38] grafana: Add tree service row Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 666 +++++++++++++++--- 1 file changed, 575 insertions(+), 91 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 62de402..df05dc0 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -244,7 +244,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "95%-tile latency of the requests.", + "description": "Latency of the requests for selected quantile", "fieldConfig": { "defaults": { "color": { @@ -326,7 +326,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -335,109 +335,560 @@ "refId": "A" } ], - "title": "Latency, 0.95q", + "title": "Latency", "transformations": [], "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "99%-tile latency of the requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "collapsed": true, "gridPos": { - "h": 10, + "h": 1, "w": 24, "x": 0, "y": 31 }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ + "id": 6, + "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" + "description": "Requests per second by Pilorama method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed pilorama requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the pilorama requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Duration of waiting time for replication, replication task duration and synchronization for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_replicate_wait_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "replicate wait success:{{success}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile,\n sum(rate(frostfs_node_treeservice_replicate_task_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "replicate task success:{{success}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_sync_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "sync success:{{success}}", + "range": true, + "refId": "C" + } + ], + "title": "Replication and synchronization", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Pilorama mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_pilorama_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama mode", + "transformations": [], + "type": "timeseries" } ], - "title": "Latency, 0.99q", - "transformations": [], - "type": "timeseries" + "title": "Tree service", + "type": "row" } ], "refresh": "1m", @@ -459,7 +910,7 @@ "definition": "label_values(frostfs_node_state_health,instance)", "hide": 0, "includeAll": false, - "label": "", + "label": "Instance", "multi": false, "name": "instance", "options": [], @@ -472,11 +923,44 @@ "skipUrlSync": false, "sort": 1, "type": "query" + }, + { + "current": { + "selected": false, + "text": "0.95", + "value": "0.95" + }, + "hide": 0, + "includeAll": false, + "label": "Quantile", + "multi": false, + "name": "quantile", + "options": [ + { + "selected": true, + "text": "0.95", + "value": "0.95" + }, + { + "selected": false, + "text": "0.99", + "value": "0.99" + }, + { + "selected": false, + "text": "1.00", + "value": "1.00" + } + ], + "query": "0.95, 0.99, 1.00", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" } ] }, "time": { - "from": "now-30m", + "from": "now-1h", "to": "now" }, "timepicker": {}, From 4f9285251ffcf271861f44c6c05dc2359c3931c3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Aug 2023 15:06:03 +0300 Subject: [PATCH 08/81] [#38] grafana: Add writecache boards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 727 +++++++++++++++++- 1 file changed, 724 insertions(+), 3 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index df05dc0..ce27576 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -354,7 +354,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Requests per second by Pilorama method for each shard", + "description": "Requests per second by pilorama method for each shard", "fieldConfig": { "defaults": { "color": { @@ -640,7 +640,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method, le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -833,7 +833,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -889,6 +890,726 @@ ], "title": "Tree service", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by writecache method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed writecache requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the Writecache requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Actual count of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Actual count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Estimated size of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Estimated size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the writecache background operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operations rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Writecache mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Writecache", + "type": "row" } ], "refresh": "1m", From d5ee2907403686eb8df604bec19c9d6f05f94c4b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Aug 2023 16:14:29 +0300 Subject: [PATCH 09/81] [#38] grafana: Add node instance state Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 746 +++++++++++------- 1 file changed, 446 insertions(+), 300 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index ce27576..4bd7a0e 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -23,93 +23,102 @@ "liveNow": false, "panels": [ { - "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Instance state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Undefined" + }, + "1": { + "index": 1, + "text": "Starting" + }, + "2": { + "index": 2, + "text": "Ready" + }, + "3": { + "index": 3, + "text": "Shutting down" + } + }, + "type": "value" + } + ], + "max": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, "gridPos": { - "h": 1, + "h": 6, "w": 24, "x": 0, "y": 0 }, - "id": 2, - "panels": [], - "title": "Server", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 1, + "id": 21, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -125,217 +134,16 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "frostfs_node_state_health{job=\"node\", instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "legendFormat": "state", "range": true, "refId": "A" } ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", + "title": "State", "transformations": [], "type": "timeseries" }, @@ -345,7 +153,350 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 6 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Server", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 }, "id": 6, "panels": [ @@ -398,8 +549,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -499,8 +649,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -599,8 +748,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -699,8 +847,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -833,8 +980,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -897,7 +1043,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 8 }, "id": 12, "panels": [ @@ -967,7 +1113,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 39 }, "id": 13, "options": { @@ -1068,7 +1214,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 49 }, "id": 15, "options": { @@ -1168,7 +1314,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 59 }, "id": 14, "options": { @@ -1268,7 +1414,7 @@ "h": 10, "w": 24, "x": 0, - "y": 63 + "y": 69 }, "id": 16, "options": { @@ -1368,7 +1514,7 @@ "h": 10, "w": 24, "x": 0, - "y": 73 + "y": 79 }, "id": 17, "options": { @@ -1468,7 +1614,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 89 }, "id": 18, "options": { @@ -1570,7 +1716,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 99 }, "id": 19, "options": { From d76cc2e48a0b61534e024707d038184881ba5e5b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:31:19 +0300 Subject: [PATCH 10/81] [#38] grafana: Add replicator dashbords Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 982 ++++++++++++------ 1 file changed, 654 insertions(+), 328 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 4bd7a0e..0b97983 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -22,6 +22,19 @@ "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "Health", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -110,7 +123,7 @@ "h": 6, "w": 24, "x": 0, - "y": 0 + "y": 1 }, "id": 21, "options": { @@ -148,347 +161,346 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 7 }, "id": 2, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "neo.fs.v2.object.ObjectService.Search" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" + "unit": "reqps" }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" } ], - "title": "Server", - "type": "row" + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" }, { "collapsed": true, @@ -496,7 +508,7 @@ "h": 1, "w": 24, "x": 0, - "y": 7 + "y": 38 }, "id": 6, "panels": [ @@ -1043,7 +1055,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 39 }, "id": 12, "panels": [ @@ -1113,7 +1125,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 4 }, "id": 13, "options": { @@ -1214,7 +1226,7 @@ "h": 10, "w": 24, "x": 0, - "y": 49 + "y": 14 }, "id": 15, "options": { @@ -1314,7 +1326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 59 + "y": 24 }, "id": 14, "options": { @@ -1414,7 +1426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 69 + "y": 34 }, "id": 16, "options": { @@ -1514,7 +1526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 79 + "y": 44 }, "id": 17, "options": { @@ -1614,7 +1626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 89 + "y": 54 }, "id": 18, "options": { @@ -1716,7 +1728,7 @@ "h": 10, "w": 24, "x": 0, - "y": 99 + "y": 64 }, "id": 19, "options": { @@ -1756,6 +1768,320 @@ ], "title": "Writecache", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of in-flight requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "in-flight requests", + "range": true, + "refId": "A" + } + ], + "title": "In-flight request", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "object processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Objects processing rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects payload replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "payload processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Payload processing rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Replicator", + "type": "row" } ], "refresh": "1m", From 74d43f48f2c31ea20d4238efd9a24218239077c4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:48:53 +0300 Subject: [PATCH 11/81] [#38] grafana: Add object service dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 435 +++++++++++++++++- 1 file changed, 428 insertions(+), 7 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0b97983..264ad20 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -1125,7 +1125,7 @@ "h": 10, "w": 24, "x": 0, - "y": 4 + "y": 40 }, "id": 13, "options": { @@ -1226,7 +1226,7 @@ "h": 10, "w": 24, "x": 0, - "y": 14 + "y": 50 }, "id": 15, "options": { @@ -1326,7 +1326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 24 + "y": 60 }, "id": 14, "options": { @@ -1426,7 +1426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 34 + "y": 70 }, "id": 16, "options": { @@ -1526,7 +1526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 44 + "y": 80 }, "id": 17, "options": { @@ -1626,7 +1626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 54 + "y": 90 }, "id": 18, "options": { @@ -1728,7 +1728,7 @@ "h": 10, "w": 24, "x": 0, - "y": 64 + "y": 100 }, "id": 19, "options": { @@ -2082,6 +2082,427 @@ ], "title": "Replicator", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 112 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed object service requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 122 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the object service requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 132 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 142 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Object service", + "type": "row" } ], "refresh": "1m", From 201855e72958978118c8ff9e2aede2c757acf5bc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:57:07 +0300 Subject: [PATCH 12/81] [#38] grafana: Add epoch dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 108 ++++++++++++++---- 1 file changed, 87 insertions(+), 21 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 264ad20..4aaaebf 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -32,7 +32,7 @@ }, "id": 22, "panels": [], - "title": "Health", + "title": "General", "type": "row" }, { @@ -121,7 +121,7 @@ }, "gridPos": { "h": 6, - "w": 24, + "w": 12, "x": 0, "y": 1 }, @@ -160,6 +160,72 @@ "transformations": [], "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Current epoch on the instance", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 32, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_ir_epoch{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Epoch", + "type": "gauge" + }, { "collapsed": false, "gridPos": { @@ -577,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 40 }, "id": 8, "options": { @@ -677,7 +743,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 50 }, "id": 9, "options": { @@ -776,7 +842,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 60 }, "id": 10, "options": { @@ -875,7 +941,7 @@ "h": 10, "w": 24, "x": 0, - "y": 62 + "y": 70 }, "id": 7, "options": { @@ -1008,7 +1074,7 @@ "h": 10, "w": 24, "x": 0, - "y": 72 + "y": 80 }, "id": 11, "options": { @@ -1125,7 +1191,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 48 }, "id": 13, "options": { @@ -1226,7 +1292,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 58 }, "id": 15, "options": { @@ -1326,7 +1392,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 68 }, "id": 14, "options": { @@ -1426,7 +1492,7 @@ "h": 10, "w": 24, "x": 0, - "y": 70 + "y": 78 }, "id": 16, "options": { @@ -1526,7 +1592,7 @@ "h": 10, "w": 24, "x": 0, - "y": 80 + "y": 88 }, "id": 17, "options": { @@ -1626,7 +1692,7 @@ "h": 10, "w": 24, "x": 0, - "y": 90 + "y": 98 }, "id": 18, "options": { @@ -1728,7 +1794,7 @@ "h": 10, "w": 24, "x": 0, - "y": 100 + "y": 108 }, "id": 19, "options": { @@ -1842,7 +1908,7 @@ "h": 10, "w": 24, "x": 0, - "y": 75 + "y": 83 }, "id": 24, "options": { @@ -1942,7 +2008,7 @@ "h": 10, "w": 24, "x": 0, - "y": 85 + "y": 93 }, "id": 25, "options": { @@ -2042,7 +2108,7 @@ "h": 10, "w": 24, "x": 0, - "y": 95 + "y": 103 }, "id": 26, "options": { @@ -2159,7 +2225,7 @@ "h": 10, "w": 24, "x": 0, - "y": 112 + "y": 120 }, "id": 28, "options": { @@ -2260,7 +2326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 122 + "y": 130 }, "id": 29, "options": { @@ -2360,7 +2426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 132 + "y": 140 }, "id": 30, "options": { @@ -2463,7 +2529,7 @@ "h": 10, "w": 24, "x": 0, - "y": 142 + "y": 150 }, "id": 31, "options": { From 8218440525c4129ef97e18534522a879a7a909dd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 16:11:35 +0300 Subject: [PATCH 13/81] [#38] grafana: Add metabase dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 3332 ++++++++++------- 1 file changed, 1876 insertions(+), 1456 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 4aaaebf..20a3ed2 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -576,6 +576,1881 @@ "x": 0, "y": 38 }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 120 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed object service requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 130 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the object service requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 140 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 150 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Object service", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 33, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by metabase method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed metabase requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_metabase_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Metabase mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_metabase_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Metabase", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by writecache method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed writecache requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the writecache requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Actual count of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Actual count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Estimated size of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Estimated size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the writecache background operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operations rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Writecache mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Writecache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of in-flight requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "in-flight requests", + "range": true, + "refId": "A" + } + ], + "title": "In-flight request", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "object processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Objects processing rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects payload replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "payload processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Payload processing rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Replicator", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, "id": 6, "panels": [ { @@ -1114,1461 +2989,6 @@ ], "title": "Tree service", "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 12, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by writecache method for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 13, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed writecache requests for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 58 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the Writecache requests for selected quantile for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Actual count of the objects, that writecache stores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 78 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}", - "range": true, - "refId": "A" - } - ], - "title": "Actual count", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Estimated size of the objects, that writecache stores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 88 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}", - "range": true, - "refId": "A" - } - ], - "title": "Estimated size", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the writecache background operations", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 98 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", - "range": true, - "refId": "A" - } - ], - "title": "Operations rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Writecache mode for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1.5, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 108 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, mode {{mode}}", - "range": true, - "refId": "A" - } - ], - "title": "Mode", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Writecache", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 23, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Count of in-flight requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 83 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "in-flight requests", - "range": true, - "refId": "A" - } - ], - "title": "In-flight request", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the objects replicated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 93 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "object processing rate", - "range": true, - "refId": "A" - } - ], - "title": "Objects processing rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the objects payload replicated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "payload processing rate", - "range": true, - "refId": "A" - } - ], - "title": "Payload processing rate", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Replicator", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 27, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by object service method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 120 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed object service requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 130 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the object service requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 140 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Bytes per second by object service method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 150 - }, - "id": 31, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Payload rate", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Object service", - "type": "row" } ], "refresh": "1m", @@ -2579,7 +2999,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 1408558631b490387e17474775a704cdd1c9e4c1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 13:59:12 +0300 Subject: [PATCH 14/81] [#38] grafana: Fix units Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 681 +++++++++--------- 1 file changed, 341 insertions(+), 340 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 20a3ed2..a2a37aa 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -227,7 +227,7 @@ "type": "gauge" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -235,338 +235,339 @@ "y": 7 }, "id": 2, - "panels": [], - "title": "Server", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "neo.fs.v2.object.ObjectService.Search" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "desc" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" } ], - "title": "Latency", - "transformations": [], - "type": "timeseries" + "title": "Server", + "type": "row" }, { "collapsed": true, @@ -574,7 +575,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 8 }, "id": 27, "panels": [ @@ -644,7 +645,7 @@ "h": 10, "w": 24, "x": 0, - "y": 120 + "y": 9 }, "id": 28, "options": { @@ -745,7 +746,7 @@ "h": 10, "w": 24, "x": 0, - "y": 130 + "y": 19 }, "id": 29, "options": { @@ -837,7 +838,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -845,7 +846,7 @@ "h": 10, "w": 24, "x": 0, - "y": 140 + "y": 29 }, "id": 30, "options": { @@ -940,7 +941,7 @@ } ] }, - "unit": "Bps" + "unit": "binBps" }, "overrides": [] }, @@ -948,7 +949,7 @@ "h": 10, "w": 24, "x": 0, - "y": 150 + "y": 39 }, "id": 31, "options": { @@ -995,7 +996,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 9 }, "id": 33, "panels": [ @@ -1065,7 +1066,7 @@ "h": 10, "w": 24, "x": 0, - "y": 1 + "y": 10 }, "id": 35, "options": { @@ -1166,7 +1167,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 20 }, "id": 37, "options": { @@ -1258,7 +1259,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -1266,7 +1267,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 30 }, "id": 36, "options": { @@ -1368,7 +1369,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 40 }, "id": 34, "options": { @@ -1415,7 +1416,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 10 }, "id": 12, "panels": [ @@ -1485,7 +1486,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 11 }, "id": 13, "options": { @@ -1586,7 +1587,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 21 }, "id": 15, "options": { @@ -1678,7 +1679,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -1686,7 +1687,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 31 }, "id": 14, "options": { @@ -1786,7 +1787,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 41 }, "id": 16, "options": { @@ -1878,7 +1879,7 @@ } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, @@ -1886,7 +1887,7 @@ "h": 10, "w": 24, "x": 0, - "y": 81 + "y": 51 }, "id": 17, "options": { @@ -1986,7 +1987,7 @@ "h": 10, "w": 24, "x": 0, - "y": 91 + "y": 61 }, "id": 18, "options": { @@ -2088,7 +2089,7 @@ "h": 10, "w": 24, "x": 0, - "y": 101 + "y": 71 }, "id": 19, "options": { @@ -2135,7 +2136,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 11 }, "id": 23, "panels": [ @@ -2202,7 +2203,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 12 }, "id": 24, "options": { @@ -2302,7 +2303,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 22 }, "id": 25, "options": { @@ -2394,7 +2395,7 @@ } ] }, - "unit": "Bps" + "unit": "binBps" }, "overrides": [] }, @@ -2402,7 +2403,7 @@ "h": 10, "w": 24, "x": 0, - "y": 103 + "y": 32 }, "id": 26, "options": { @@ -2449,7 +2450,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 12 }, "id": 6, "panels": [ @@ -2999,7 +3000,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 0231b2bbf0b0a2cf95b255839c79bbbbc28527e7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 14:34:54 +0300 Subject: [PATCH 15/81] [#38] grafana: Add blobstore dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 565 +++++++++++++++++- 1 file changed, 551 insertions(+), 14 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index a2a37aa..0d8213f 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -168,10 +168,6 @@ "description": "Current epoch on the instance", "fieldConfig": { "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, "mappings": [], "thresholds": { "mode": "absolute", @@ -186,7 +182,7 @@ } ] }, - "unit": "none" + "unit": "short" }, "overrides": [] }, @@ -198,6 +194,9 @@ }, "id": 32, "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -206,8 +205,7 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "textMode": "auto" }, "pluginVersion": "9.5.6", "targets": [ @@ -224,7 +222,7 @@ } ], "title": "Epoch", - "type": "gauge" + "type": "stat" }, { "collapsed": true, @@ -1066,7 +1064,7 @@ "h": 10, "w": 24, "x": 0, - "y": 10 + "y": 51 }, "id": 35, "options": { @@ -1167,7 +1165,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 61 }, "id": 37, "options": { @@ -1267,7 +1265,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 71 }, "id": 36, "options": { @@ -1369,7 +1367,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 81 }, "id": 34, "options": { @@ -2138,6 +2136,545 @@ "x": 0, "y": 11 }, + "id": 38, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobstore method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobstore requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobstore_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobstore shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobstore mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobstore_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobstore", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, "id": 23, "panels": [ { @@ -2450,7 +2987,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 6, "panels": [ @@ -3000,7 +3537,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From acd32cb87720840754556a255b15f81c6faf6fb8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 14:50:04 +0300 Subject: [PATCH 16/81] [#38] grafana: Add blobovnizca dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 757 +++++++++++++++++- 1 file changed, 751 insertions(+), 6 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d8213f..cd8f32c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -2204,7 +2204,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 39, "options": { @@ -2305,7 +2305,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 40, "options": { @@ -2405,7 +2405,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 41, "options": { @@ -2508,7 +2508,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 43, "options": { @@ -2626,7 +2626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "id": 42, "options": { @@ -2675,6 +2675,751 @@ "x": 0, "y": 12 }, + "id": 44, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobovnizca method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobovnizca requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the blobovnizca requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovniza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobovnizca shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobovnizca size for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovniza_tree_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of open DB's of blobovnizca for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovniza_tree_open_blobovnizca_count{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open DB count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobovnizca mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobovniza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobovnizca", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "id": 23, "panels": [ { @@ -2987,7 +3732,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 6, "panels": [ From cc963b78b5ca1f4b16c67d516a18467954dcce63 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 15:03:48 +0300 Subject: [PATCH 17/81] [#38] grafana: Add fstree dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 567 +++++++++++++++++- 1 file changed, 553 insertions(+), 14 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index cd8f32c..6fd6bf5 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -2204,7 +2204,7 @@ "h": 10, "w": 24, "x": 0, - "y": 13 + "y": 12 }, "id": 39, "options": { @@ -2305,7 +2305,7 @@ "h": 10, "w": 24, "x": 0, - "y": 23 + "y": 22 }, "id": 40, "options": { @@ -2405,7 +2405,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 32 }, "id": 41, "options": { @@ -2508,7 +2508,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 42 }, "id": 43, "options": { @@ -2626,7 +2626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 52 }, "id": 42, "options": { @@ -2675,6 +2675,545 @@ "x": 0, "y": 12 }, + "id": 52, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by fstree method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed fstree requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the fstree requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_fstree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by fstree shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "FSTree mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_fstree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "FSTree", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "id": 44, "panels": [ { @@ -2743,7 +3282,7 @@ "h": 10, "w": 24, "x": 0, - "y": 1 + "y": 64 }, "id": 45, "options": { @@ -2844,7 +3383,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 74 }, "id": 46, "options": { @@ -2944,7 +3483,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 84 }, "id": 47, "options": { @@ -3047,7 +3586,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 94 }, "id": 48, "options": { @@ -3166,7 +3705,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 104 }, "id": 50, "options": { @@ -3269,7 +3808,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 114 }, "id": 51, "options": { @@ -3371,7 +3910,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 124 }, "id": 49, "options": { @@ -3418,7 +3957,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 23, "panels": [ @@ -3732,7 +4271,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 6, "panels": [ From d8df46b4d19ae673d44a0e55585c1a3fa68f8787 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sat, 5 Aug 2023 21:16:47 +0300 Subject: [PATCH 18/81] [#38] grafana: Add engine dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 743 ++++++++++++++++-- 1 file changed, 672 insertions(+), 71 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 6fd6bf5..0d08ded 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -403,7 +403,7 @@ "options": { "mode": "exclude", "names": [ - "neo.fs.v2.object.ObjectService.Search" + "neo.fs.v2.object.ObjectService.Put" ], "prefix": "All except:", "readOnly": true @@ -643,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 20 }, "id": 28, "options": { @@ -744,7 +744,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 30 }, "id": 29, "options": { @@ -844,7 +844,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 40 }, "id": 30, "options": { @@ -930,8 +930,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -947,7 +946,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 50 }, "id": 31, "options": { @@ -996,6 +995,634 @@ "x": 0, "y": 9 }, + "id": 58, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by storage engine service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_engine_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the engine requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_engine_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Payload size by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_payload_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Objects count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, type {{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Errors count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_errors_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Errors count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_engine_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Shard mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Engine", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, "id": 33, "panels": [ { @@ -1047,8 +1674,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1148,8 +1774,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1248,8 +1873,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1350,8 +1974,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1414,7 +2037,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 12, "panels": [ @@ -1467,8 +2090,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1568,8 +2190,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1668,8 +2289,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1768,8 +2388,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1868,8 +2487,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1968,8 +2586,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2070,8 +2687,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2134,7 +2750,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 38, "panels": [ @@ -2204,7 +2820,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 39, "options": { @@ -2305,7 +2921,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 40, "options": { @@ -2405,7 +3021,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 41, "options": { @@ -2508,7 +3124,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 43, "options": { @@ -2626,7 +3242,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "id": 42, "options": { @@ -2673,7 +3289,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 52, "panels": [ @@ -2726,8 +3342,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2827,8 +3442,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2927,8 +3541,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3030,8 +3643,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3148,8 +3760,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3212,7 +3823,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 44, "panels": [ @@ -3265,8 +3876,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3366,8 +3976,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3466,8 +4075,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3569,8 +4177,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3688,8 +4295,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3791,8 +4397,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3893,8 +4498,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3957,7 +4561,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 23, "panels": [ @@ -4007,8 +4611,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4107,8 +4710,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4207,8 +4809,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4271,7 +4872,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 6, "panels": [ @@ -4821,7 +5422,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From b6f47cb2c260b7c12377f1cecddee42cb105e05a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sun, 6 Aug 2023 13:44:37 +0300 Subject: [PATCH 19/81] [#38] grafana: Add GC dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 180 +++++++++++++++--- 1 file changed, 152 insertions(+), 28 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d08ded..0d9b437 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -283,8 +283,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -384,8 +383,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -509,8 +507,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -643,7 +640,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 9 }, "id": 28, "options": { @@ -744,7 +741,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 19 }, "id": 29, "options": { @@ -827,8 +824,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -844,7 +840,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 29 }, "id": 30, "options": { @@ -946,7 +942,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 39 }, "id": 31, "options": { @@ -1063,7 +1059,7 @@ "h": 10, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 59, "options": { @@ -1163,7 +1159,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 21 }, "id": 60, "options": { @@ -1266,7 +1262,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 31 }, "id": 61, "options": { @@ -1369,7 +1365,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 41 }, "id": 62, "options": { @@ -1472,7 +1468,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 51 }, "id": 63, "options": { @@ -1574,7 +1570,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 61 }, "id": 64, "options": { @@ -2803,8 +2799,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2904,8 +2899,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3004,8 +2998,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3107,8 +3100,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3225,8 +3217,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5412,6 +5403,139 @@ ], "title": "Tree service", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 66, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of objects GC marked to remove, deleted or failed to delete from disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_garbage_collector_deleted_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_garbage_collector_marked_for_removal_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, marked for remove, {{object_type}}", + "range": true, + "refId": "B" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + } + ], + "title": "GC", + "type": "row" } ], "refresh": "1m", From 9bb0385b85e20a8149c599c1fbdefcdf7143e737 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sun, 6 Aug 2023 15:15:54 +0300 Subject: [PATCH 20/81] [#38] grafana: Add morph dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 999 ++++++++++++++++-- 1 file changed, 904 insertions(+), 95 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d9b437..f7d827c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -182,7 +182,7 @@ } ] }, - "unit": "short" + "unit": "none" }, "overrides": [] }, @@ -283,7 +283,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -299,7 +300,7 @@ "h": 10, "w": 24, "x": 0, - "y": 8 + "y": 2 }, "id": 1, "options": { @@ -383,7 +384,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -424,7 +426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 12 }, "id": 3, "options": { @@ -507,7 +509,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -523,7 +526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 28 + "y": 22 }, "id": 4, "options": { @@ -640,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 3 }, "id": 28, "options": { @@ -741,7 +744,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 13 }, "id": 29, "options": { @@ -824,7 +827,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -840,7 +844,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 23 }, "id": 30, "options": { @@ -926,7 +930,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -942,7 +947,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 33 }, "id": 31, "options": { @@ -1059,7 +1064,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 4 }, "id": 59, "options": { @@ -1159,7 +1164,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 14 }, "id": 60, "options": { @@ -1262,7 +1267,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 24 }, "id": 61, "options": { @@ -1365,7 +1370,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 34 }, "id": 62, "options": { @@ -1468,7 +1473,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 44 }, "id": 63, "options": { @@ -1570,7 +1575,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 54 }, "id": 64, "options": { @@ -1670,7 +1675,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1686,7 +1692,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 5 }, "id": 35, "options": { @@ -1770,7 +1776,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1786,7 +1793,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 15 }, "id": 37, "options": { @@ -1869,7 +1876,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1885,7 +1893,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 25 }, "id": 36, "options": { @@ -1970,7 +1978,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1986,7 +1995,7 @@ "h": 10, "w": 24, "x": 0, - "y": 81 + "y": 35 }, "id": 34, "options": { @@ -2086,7 +2095,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2102,7 +2112,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 6 }, "id": 13, "options": { @@ -2186,7 +2196,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2202,7 +2213,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 16 }, "id": 15, "options": { @@ -2285,7 +2296,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2301,7 +2313,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 26 }, "id": 14, "options": { @@ -2384,7 +2396,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2400,7 +2413,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 36 }, "id": 16, "options": { @@ -2483,7 +2496,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2499,7 +2513,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 46 }, "id": 17, "options": { @@ -2582,7 +2596,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2590,7 +2605,7 @@ } ] }, - "unit": "none" + "unit": "ops" }, "overrides": [] }, @@ -2598,7 +2613,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 56 }, "id": 18, "options": { @@ -2683,7 +2698,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2699,7 +2715,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 66 }, "id": 19, "options": { @@ -2799,7 +2815,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2815,7 +2832,7 @@ "h": 10, "w": 24, "x": 0, - "y": 13 + "y": 7 }, "id": 39, "options": { @@ -2899,7 +2916,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2915,7 +2933,7 @@ "h": 10, "w": 24, "x": 0, - "y": 23 + "y": 17 }, "id": 40, "options": { @@ -2998,7 +3016,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3014,7 +3033,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 27 }, "id": 41, "options": { @@ -3100,7 +3119,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3116,7 +3136,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 37 }, "id": 43, "options": { @@ -3217,7 +3237,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3233,7 +3254,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 47 }, "id": 42, "options": { @@ -3333,7 +3354,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3349,7 +3371,7 @@ "h": 10, "w": 24, "x": 0, - "y": 63 + "y": 8 }, "id": 53, "options": { @@ -3433,7 +3455,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3449,7 +3472,7 @@ "h": 10, "w": 24, "x": 0, - "y": 73 + "y": 18 }, "id": 54, "options": { @@ -3532,7 +3555,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3548,7 +3572,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 28 }, "id": 55, "options": { @@ -3634,7 +3658,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3650,7 +3675,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 38 }, "id": 56, "options": { @@ -3751,7 +3776,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3767,7 +3793,7 @@ "h": 10, "w": 24, "x": 0, - "y": 103 + "y": 48 }, "id": 57, "options": { @@ -3867,7 +3893,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3883,7 +3910,7 @@ "h": 10, "w": 24, "x": 0, - "y": 64 + "y": 9 }, "id": 45, "options": { @@ -3967,7 +3994,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3983,7 +4011,7 @@ "h": 10, "w": 24, "x": 0, - "y": 74 + "y": 19 }, "id": 46, "options": { @@ -4066,7 +4094,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4082,7 +4111,7 @@ "h": 10, "w": 24, "x": 0, - "y": 84 + "y": 29 }, "id": 47, "options": { @@ -4168,7 +4197,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4184,7 +4214,7 @@ "h": 10, "w": 24, "x": 0, - "y": 94 + "y": 39 }, "id": 48, "options": { @@ -4286,7 +4316,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4302,7 +4333,7 @@ "h": 10, "w": 24, "x": 0, - "y": 104 + "y": 49 }, "id": 50, "options": { @@ -4388,7 +4419,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4404,7 +4436,7 @@ "h": 10, "w": 24, "x": 0, - "y": 114 + "y": 59 }, "id": 51, "options": { @@ -4489,7 +4521,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4505,7 +4538,7 @@ "h": 10, "w": 24, "x": 0, - "y": 124 + "y": 69 }, "id": 49, "options": { @@ -4602,7 +4635,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4618,7 +4652,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 10 }, "id": 24, "options": { @@ -4701,7 +4735,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4709,7 +4744,7 @@ } ] }, - "unit": "none" + "unit": "cps" }, "overrides": [] }, @@ -4717,7 +4752,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 20 }, "id": 25, "options": { @@ -4800,7 +4835,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4816,7 +4852,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 30 }, "id": 26, "options": { @@ -4916,7 +4952,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4932,7 +4969,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 11 }, "id": 8, "options": { @@ -5016,7 +5053,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5032,7 +5070,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 21 }, "id": 9, "options": { @@ -5115,7 +5153,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5131,7 +5170,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 31 }, "id": 10, "options": { @@ -5214,7 +5253,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5230,7 +5270,7 @@ "h": 10, "w": 24, "x": 0, - "y": 70 + "y": 41 }, "id": 7, "options": { @@ -5347,7 +5387,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5363,7 +5404,7 @@ "h": 10, "w": 24, "x": 0, - "y": 80 + "y": 51 }, "id": 11, "options": { @@ -5480,7 +5521,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 12 }, "id": 65, "options": { @@ -5536,6 +5577,774 @@ ], "title": "GC", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 67, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of endpoint switches", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 68, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_morph_switches_total{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Endpoint switches", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Index of the last received block", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 69, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_morph_last_block{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Last block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of notifications received by notification type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_morph_notifications_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{notification_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Notifications rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of of contract invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract method invoke rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed contract invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Put" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\", success!=\"true\"}[$__rate_interval])\n)\n / \nsum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract invocations error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the contract invocations for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_morph_invoke_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (contract, invoke_type, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract invocations latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of of morph cache methods invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_morphcache_request_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache method rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the morph cache for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_morphcache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache invocations latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Morph", + "type": "row" } ], "refresh": "1m", @@ -5546,7 +6355,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 04260ad0d8eba6dd33cf181371be3fa83996ed9c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 7 Aug 2023 12:17:06 +0300 Subject: [PATCH 21/81] [#38] grafana: Fix writecache boards Stack count and size dashborads. Signed-off-by: Dmitrii Stepanov --- .gitattributes | 1 + .../provisioning/dashboards/storage-node.json | 161 +++++++----------- 2 files changed, 66 insertions(+), 96 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5436e42 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +/services/grafana/provisioning/dashboards/* -diff -merge \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index f7d827c..e0cd8a8 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -300,7 +300,7 @@ "h": 10, "w": 24, "x": 0, - "y": 2 + "y": 8 }, "id": 1, "options": { @@ -426,7 +426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 18 }, "id": 3, "options": { @@ -526,7 +526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 28 }, "id": 4, "options": { @@ -626,8 +626,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -727,8 +726,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -827,8 +825,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -930,8 +927,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1047,8 +1043,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1147,8 +1142,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1250,8 +1244,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1353,8 +1346,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1456,8 +1448,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1558,8 +1549,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1675,8 +1665,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1776,8 +1765,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1876,8 +1864,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1978,8 +1965,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2112,7 +2098,7 @@ "h": 10, "w": 24, "x": 0, - "y": 6 + "y": 12 }, "id": 13, "options": { @@ -2213,7 +2199,7 @@ "h": 10, "w": 24, "x": 0, - "y": 16 + "y": 22 }, "id": 15, "options": { @@ -2313,7 +2299,7 @@ "h": 10, "w": 24, "x": 0, - "y": 26 + "y": 32 }, "id": 14, "options": { @@ -2368,7 +2354,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2385,7 +2371,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2413,7 +2399,7 @@ "h": 10, "w": 24, "x": 0, - "y": 36 + "y": 42 }, "id": 16, "options": { @@ -2468,7 +2454,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2485,7 +2471,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2513,7 +2499,7 @@ "h": 10, "w": 24, "x": 0, - "y": 46 + "y": 52 }, "id": 17, "options": { @@ -2568,7 +2554,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2585,7 +2571,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2613,7 +2599,7 @@ "h": 10, "w": 24, "x": 0, - "y": 56 + "y": 62 }, "id": 18, "options": { @@ -2715,7 +2701,7 @@ "h": 10, "w": 24, "x": 0, - "y": 66 + "y": 72 }, "id": 19, "options": { @@ -2832,7 +2818,7 @@ "h": 10, "w": 24, "x": 0, - "y": 7 + "y": 83 }, "id": 39, "options": { @@ -2933,7 +2919,7 @@ "h": 10, "w": 24, "x": 0, - "y": 17 + "y": 93 }, "id": 40, "options": { @@ -3033,7 +3019,7 @@ "h": 10, "w": 24, "x": 0, - "y": 27 + "y": 103 }, "id": 41, "options": { @@ -3136,7 +3122,7 @@ "h": 10, "w": 24, "x": 0, - "y": 37 + "y": 113 }, "id": 43, "options": { @@ -3254,7 +3240,7 @@ "h": 10, "w": 24, "x": 0, - "y": 47 + "y": 123 }, "id": 42, "options": { @@ -3371,7 +3357,7 @@ "h": 10, "w": 24, "x": 0, - "y": 8 + "y": 84 }, "id": 53, "options": { @@ -3472,7 +3458,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 94 }, "id": 54, "options": { @@ -3572,7 +3558,7 @@ "h": 10, "w": 24, "x": 0, - "y": 28 + "y": 104 }, "id": 55, "options": { @@ -3675,7 +3661,7 @@ "h": 10, "w": 24, "x": 0, - "y": 38 + "y": 114 }, "id": 56, "options": { @@ -3793,7 +3779,7 @@ "h": 10, "w": 24, "x": 0, - "y": 48 + "y": 124 }, "id": 57, "options": { @@ -3910,7 +3896,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 85 }, "id": 45, "options": { @@ -4011,7 +3997,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 95 }, "id": 46, "options": { @@ -4111,7 +4097,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 105 }, "id": 47, "options": { @@ -4214,7 +4200,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 115 }, "id": 48, "options": { @@ -4333,7 +4319,7 @@ "h": 10, "w": 24, "x": 0, - "y": 49 + "y": 125 }, "id": 50, "options": { @@ -4436,7 +4422,7 @@ "h": 10, "w": 24, "x": 0, - "y": 59 + "y": 135 }, "id": 51, "options": { @@ -4538,7 +4524,7 @@ "h": 10, "w": 24, "x": 0, - "y": 69 + "y": 145 }, "id": 49, "options": { @@ -4635,8 +4621,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4735,8 +4720,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4835,8 +4819,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4952,8 +4935,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5053,8 +5035,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5153,8 +5134,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5253,8 +5233,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5387,8 +5366,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5504,8 +5482,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5601,8 +5578,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5665,8 +5641,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5762,8 +5737,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5862,8 +5836,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5963,8 +5936,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6088,8 +6060,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6188,8 +6159,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6288,8 +6258,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6355,7 +6324,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 51053e331720b36b3b52adb8420b3883f530ce6f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 11 Aug 2023 15:57:45 +0300 Subject: [PATCH 22/81] [#46] .forgejo: Update DCO action Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/dco.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 880addd..6746408 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -1,3 +1,4 @@ +name: DCO action on: [pull_request] jobs: @@ -12,9 +13,9 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' - name: Run commit format checker - uses: https://git.alexvan.in/alexvanin/dco-go@v1 + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2 with: - from: dca6ff62 + from: 'origin/${{ github.event.pull_request.base.ref }}' From 731976cc57de38e699032428695d0a276b6ecc13 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Aug 2023 15:48:57 +0300 Subject: [PATCH 23/81] [#44] grafana: Add Client dashboard Client dashboard shows outgoing requests. Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 410 +++++++++++++++--- 1 file changed, 352 insertions(+), 58 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index e0cd8a8..f78962c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -575,6 +575,324 @@ "x": 0, "y": 8 }, + "id": 76, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_client_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_client_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_client_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_client_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 27, "panels": [ { @@ -990,7 +1308,7 @@ "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 10 }, "id": 58, "panels": [ @@ -1612,7 +1930,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 33, "panels": [ @@ -2028,7 +2346,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 12, "panels": [ @@ -2081,8 +2399,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2182,8 +2499,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2282,8 +2598,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2382,8 +2697,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2482,8 +2796,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2582,8 +2895,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2684,8 +2996,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2748,7 +3059,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 38, "panels": [ @@ -2801,8 +3112,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2902,8 +3212,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3002,8 +3311,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3105,8 +3413,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3223,8 +3530,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3287,7 +3593,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 52, "panels": [ @@ -3340,8 +3646,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3441,8 +3746,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3541,8 +3845,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3644,8 +3947,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3762,8 +4064,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3826,7 +4127,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 44, "panels": [ @@ -3879,8 +4180,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3980,8 +4280,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4080,8 +4379,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4183,8 +4481,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4302,8 +4599,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4405,8 +4701,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4507,8 +4802,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4571,7 +4865,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 23, "panels": [ @@ -4882,7 +5176,7 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 17 }, "id": 6, "panels": [ @@ -5429,7 +5723,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 18 }, "id": 66, "panels": [ @@ -5561,7 +5855,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 19 }, "id": 67, "panels": [ From 6eedab3d83db60fc68d4a7288ba463c7683b46b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Aug 2023 13:40:03 +0300 Subject: [PATCH 24/81] [#48] grafana: Fix blobovnicza board Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 199 +++++++++++++----- 1 file changed, 152 insertions(+), 47 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index f78962c..318f8c7 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -283,8 +283,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -384,8 +383,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -509,8 +507,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -626,8 +623,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -727,8 +723,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -827,8 +822,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4136,7 +4130,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Requests per second by blobovnizca method for each shard", + "description": "Requests per second by blobovnicza method for each shard", "fieldConfig": { "defaults": { "color": { @@ -4180,7 +4174,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4196,7 +4191,7 @@ "h": 10, "w": 24, "x": 0, - "y": 85 + "y": 16 }, "id": 45, "options": { @@ -4220,7 +4215,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovnicza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4238,7 +4233,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Percentage of the failed blobovnizca requests for each shard", + "description": "Percentage of the failed blobovnicza requests for each shard", "fieldConfig": { "defaults": { "color": { @@ -4280,7 +4275,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4296,7 +4292,7 @@ "h": 10, "w": 24, "x": 0, - "y": 95 + "y": 26 }, "id": 46, "options": { @@ -4320,7 +4316,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "expr": "sum(rate(frostfs_node_blobsovnicza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovnicza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", "format": "time_series", "instant": false, "interval": "", @@ -4338,7 +4334,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Latency of the blobovnizca requests for selected quantile for each shard", + "description": "Latency of the blobovnicza requests for selected quantile for each shard", "fieldConfig": { "defaults": { "color": { @@ -4379,7 +4375,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4395,7 +4392,7 @@ "h": 10, "w": 24, "x": 0, - "y": 105 + "y": 36 }, "id": 47, "options": { @@ -4419,7 +4416,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovniza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4437,7 +4434,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Bytes per second by blobovnizca shard", + "description": "Bytes per second by blobovnicza shard", "fieldConfig": { "defaults": { "color": { @@ -4481,7 +4478,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4497,7 +4495,7 @@ "h": 10, "w": 24, "x": 0, - "y": 115 + "y": 46 }, "id": 48, "options": { @@ -4521,7 +4519,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovnicza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4536,7 +4534,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovnicza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "hide": false, "instant": false, @@ -4555,7 +4553,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Blobovnizca size for each shard", + "description": "Open blobovnicza size for each shard", "fieldConfig": { "defaults": { "color": { @@ -4599,7 +4597,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4615,7 +4614,7 @@ "h": 10, "w": 24, "x": 0, - "y": 125 + "y": 56 }, "id": 50, "options": { @@ -4639,7 +4638,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "frostfs_node_blobovniza_tree_size_bytes{instance=\"$instance\"}\n", + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_size_bytes{instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", @@ -4648,7 +4647,7 @@ "refId": "A" } ], - "title": "Size", + "title": "Open blobovnicza size", "transformations": [], "type": "timeseries" }, @@ -4657,7 +4656,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Count of open DB's of blobovnizca for each shard", + "description": "Items count of open blobovnicza for each shard", "fieldConfig": { "defaults": { "color": { @@ -4701,7 +4700,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4717,7 +4717,110 @@ "h": 10, "w": 24, "x": 0, - "y": 135 + "y": 66 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_items_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open blobovnicza items", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of open blobovniczas for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 76 }, "id": 51, "options": { @@ -4741,7 +4844,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "frostfs_node_blobovniza_tree_open_blobovnizca_count{instance=\"$instance\"}\n", + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_count{instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", @@ -4750,7 +4853,7 @@ "refId": "A" } ], - "title": "Open DB count", + "title": "Open blobovnicza count", "transformations": [], "type": "timeseries" }, @@ -4759,7 +4862,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Blobovnizca mode for each shard", + "description": "Blobovnicza mode for each shard", "fieldConfig": { "defaults": { "color": { @@ -4802,7 +4905,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4818,7 +4922,7 @@ "h": 10, "w": 24, "x": 0, - "y": 145 + "y": 86 }, "id": 49, "options": { @@ -4842,7 +4946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(frostfs_node_blobovniza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "expr": "sum(frostfs_node_blobovnicza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", "format": "time_series", "instant": false, "interval": "", @@ -4856,7 +4960,7 @@ "type": "timeseries" } ], - "title": "Blobovnizca", + "title": "Blobovnicza", "type": "row" }, { @@ -5776,7 +5880,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5792,7 +5897,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 19 }, "id": 65, "options": { @@ -6618,7 +6723,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From b7ac6f30cf476556afa8f0bbb6aafecaeeb3d54f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 21 Aug 2023 10:19:37 +0300 Subject: [PATCH 25/81] [#50] services/ir: Take all contract hashes from NNS Signed-off-by: Evgenii Stratonikov --- services/ir/.ir.env | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 08d7e17..810cf6d 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -1,3 +1 @@ -FROSTFS_IR_CONTRACTS_FROSTFSID=27407c76feabc407908f3d09a3d845d45e7c981a - FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 From eba763ff79829617896ed3e3a2471e82d7538fdd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 09:54:40 +0300 Subject: [PATCH 26/81] [#49] grafana: Server bucket distribution Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/grpc-buckets.json | 362 ++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/grpc-buckets.json diff --git a/services/grafana/provisioning/dashboards/grpc-buckets.json b/services/grafana/provisioning/dashboards/grpc-buckets.json new file mode 100644 index 0000000..ba7cd5f --- /dev/null +++ b/services/grafana/provisioning/dashboards/grpc-buckets.json @@ -0,0 +1,362 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$service", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"+Inf\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "neo.fs.v2.object.ObjectService", + "value": "neo.fs.v2.object.ObjectService" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "service", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GRPC server bucket distribution", + "uid": "a6830bc6-cffa-4e29-9622-6a10921748b0", + "version": 8, + "weekStart": "" + } \ No newline at end of file From 42e634927653e641430bac1dd07ea3571003aceb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:18:39 +0300 Subject: [PATCH 27/81] [#49] grafana: Storage engine bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/engine-buckets.json | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/engine-buckets.json diff --git a/services/grafana/provisioning/dashboards/engine-buckets.json b/services/grafana/provisioning/dashboards/engine-buckets.json new file mode 100644 index 0000000..8bc704b --- /dev/null +++ b/services/grafana/provisioning/dashboards/engine-buckets.json @@ -0,0 +1,335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node engine bucket distribution", + "uid": "f886d27c-b877-495a-a1cb-d02839115cea", + "version": 2, + "weekStart": "" + } \ No newline at end of file From 4a6b481618f3eaa991a0f8db1101791136ce57ec Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:43:51 +0300 Subject: [PATCH 28/81] [#49] grafana: Writecache bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/writecache-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/writecache-buckets.json diff --git a/services/grafana/provisioning/dashboards/writecache-buckets.json b/services/grafana/provisioning/dashboards/writecache-buckets.json new file mode 100644 index 0000000..cd37893 --- /dev/null +++ b/services/grafana/provisioning/dashboards/writecache-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "BGH3NNzxkiW7JB1BKLHb5z", + "value": "BGH3NNzxkiW7JB1BKLHb5z" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Delete", + "value": "Delete" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node writecache bucket distribution", + "uid": "d2fef1d1-9ef3-437e-ad21-670a20e0e9b4", + "version": 3, + "weekStart": "" + } \ No newline at end of file From f0c3c02943fd24e14b0a63d20f8cce0ee78f7edc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:54:38 +0300 Subject: [PATCH 29/81] [#49] grafana: Blobovnicza bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/blobovnicza-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/blobovnicza-buckets.json diff --git a/services/grafana/provisioning/dashboards/blobovnicza-buckets.json b/services/grafana/provisioning/dashboards/blobovnicza-buckets.json new file mode 100644 index 0000000..e35272a --- /dev/null +++ b/services/grafana/provisioning/dashboards/blobovnicza-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node blobovnicza bucket distribution", + "uid": "e4097619-1ce9-4039-bd62-4f5b4bba8d26", + "version": 2, + "weekStart": "" + } \ No newline at end of file From fdcf71d5b6fe4d18b3060b8aec3244c19303a174 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 11:12:11 +0300 Subject: [PATCH 30/81] [#49] grafana: Add metabase bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/metabase-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/metabase-buckets.json diff --git a/services/grafana/provisioning/dashboards/metabase-buckets.json b/services/grafana/provisioning/dashboards/metabase-buckets.json new file mode 100644 index 0000000..2fba386 --- /dev/null +++ b/services/grafana/provisioning/dashboards/metabase-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 8, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Delete", + "value": "Delete" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node metabase bucket distribution", + "uid": "e110ed66-d086-461d-958a-efc6c55607d2", + "version": 1, + "weekStart": "" + } \ No newline at end of file From 90bd39d7171fb7d43c71a008e78762aaf61d8e1a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 11:23:07 +0300 Subject: [PATCH 31/81] [#43] grafana: FSTree bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/fstree-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/fstree-buckets.json diff --git a/services/grafana/provisioning/dashboards/fstree-buckets.json b/services/grafana/provisioning/dashboards/fstree-buckets.json new file mode 100644 index 0000000..151376b --- /dev/null +++ b/services/grafana/provisioning/dashboards/fstree-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Get", + "value": "Get" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node fstree bucket distribution", + "uid": "e6407e43-9419-445b-b715-284d69201519", + "version": 2, + "weekStart": "" + } \ No newline at end of file From e254eba6a86e5e0bef92e30ae33a94bdf9be7811 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 25 Aug 2023 14:09:45 +0300 Subject: [PATCH 32/81] [#52] frostfs-adm.yml: Allow maintenance mode by default It is a DEV-env after all. Signed-off-by: Evgenii Stratonikov --- frostfs-adm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/frostfs-adm.yml b/frostfs-adm.yml index c44046d..5077431 100644 --- a/frostfs-adm.yml +++ b/frostfs-adm.yml @@ -5,6 +5,7 @@ network: epoch_duration: 240 basic_income_rate: 100000000 homomorphic_hash_disabled: false + maintenance_mode_allowed: true fee: audit: 10000 candidate: 10000000000 From 4413251994bc8480c038d1acc16810eeb005de0f Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Wed, 13 Sep 2023 16:20:46 +0300 Subject: [PATCH 33/81] [#20] Add frostfs-cli configurations Signed-off-by: Marina Biryukova --- configs/s01-cli.yml | 4 ++++ configs/s02-cli.yml | 4 ++++ configs/s03-cli.yml | 4 ++++ configs/s04-cli.yml | 4 ++++ 4 files changed, 16 insertions(+) create mode 100644 configs/s01-cli.yml create mode 100644 configs/s02-cli.yml create mode 100644 configs/s03-cli.yml create mode 100644 configs/s04-cli.yml diff --git a/configs/s01-cli.yml b/configs/s01-cli.yml new file mode 100644 index 0000000..3bd7c43 --- /dev/null +++ b/configs/s01-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet01.json +password: "" +rpc-endpoint: s01.frostfs.devenv:8080 +endpoint: s01.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s02-cli.yml b/configs/s02-cli.yml new file mode 100644 index 0000000..ac5a8bb --- /dev/null +++ b/configs/s02-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet02.json +password: "" +rpc-endpoint: s02.frostfs.devenv:8080 +endpoint: s02.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s03-cli.yml b/configs/s03-cli.yml new file mode 100644 index 0000000..ed288dc --- /dev/null +++ b/configs/s03-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet03.json +password: "" +rpc-endpoint: s03.frostfs.devenv:8080 +endpoint: s03.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s04-cli.yml b/configs/s04-cli.yml new file mode 100644 index 0000000..f4353d5 --- /dev/null +++ b/configs/s04-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet04.json +password: "" +rpc-endpoint: s04.frostfs.devenv:8080 +endpoint: s04.frostfs.devenv:8081 \ No newline at end of file From c2e2b6442c970022b5dc0a7dba100840c3daa652 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 29 Aug 2023 12:45:13 +0300 Subject: [PATCH 34/81] [#47] grafana: Don't bind port to localhost Signed-off-by: Denis Kirillov --- services/grafana/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 795b87c..6d27299 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -14,8 +14,6 @@ services: - ./../../vendor/hosts:/etc/hosts - ./grafana.ini:/etc/grafana/grafana.ini - ./provisioning:/etc/grafana/provisioning - ports: - - '3000:3000' stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] From de8b58911e498b6a522297093ae221d93e2bd997 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 29 Aug 2023 14:06:09 +0300 Subject: [PATCH 35/81] [#47] prometheus: Don't bind port to localhost Signed-off-by: Denis Kirillov --- services/prometheus/docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/prometheus/docker-compose.yml b/services/prometheus/docker-compose.yml index aef2c67..e9eb4f9 100644 --- a/services/prometheus/docker-compose.yml +++ b/services/prometheus/docker-compose.yml @@ -15,8 +15,6 @@ services: - ./prometheus.yml:/etc/prometheus/prometheus.yml command: - --config.file=/etc/prometheus/prometheus.yml - ports: - - '9090:9090' stop_signal: SIGKILL env_file: [ ".env", ".prometheus.env", ".int_test.env" ] @@ -24,4 +22,4 @@ networks: prometheus_int: internet: external: true - name: basenet_internet \ No newline at end of file + name: basenet_internet From 9c9ec639f05f1479cc84be713149df0937f33ac4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 10:56:08 +0300 Subject: [PATCH 36/81] [#58] Update neo-go to v0.103.0 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- services/morph_chain/protocol.privnet.yml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.env b/.env index e04d5ca..de7b691 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.101.1 +NEOGO_VERSION=0.103.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 13e03b5..560730a 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -29,6 +29,11 @@ ApplicationConfiguration: AttemptConnPeers: 5 MinPeers: 0 Relay: true + Consensus: + Enabled: true + UnlockWallet: + Path: "./wallets/node-wallet.json" + Password: "one" RPC: Addresses: - "192.168.130.90:30333" @@ -49,6 +54,3 @@ ApplicationConfiguration: Addresses: - ":20011" Enabled: true - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" From 40454b55076c9590d9e9ded5a85069820cf88cbd Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 15 Nov 2023 16:08:15 +0300 Subject: [PATCH 37/81] [#56] Add loki Signed-off-by: Alexander Chuprov --- .env | 8 +- services/grafana/.hosts | 1 + services/grafana/docker-compose.yml | 8 + .../dashboards/storage-node-logs.json | 358 ++++++++++++++++++ .../provisioning/datasources/datasource.yml | 5 + services/storage/cfg/config.yml | 7 +- 6 files changed, 384 insertions(+), 3 deletions(-) create mode 100644 services/grafana/provisioning/dashboards/storage-node-logs.json diff --git a/.env b/.env index de7b691..0969ff5 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.103.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=365a7ca0 +IR_VERSION=0.37.0-rc.1-93-g364f835b IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=365a7ca0 +NODE_VERSION=0.37.0-rc.1-93-g364f835b NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -64,3 +64,7 @@ PROMETHEUS_IMAGE=prom/prometheus # Grafana versions GRAFANA_VERSION=9.5.6 GRAFANA_IMAGE=grafana/grafana + +# Loki versions +LOKI_VERSION=2.9.1 +LOKI_IMAGE=grafana/loki \ No newline at end of file diff --git a/services/grafana/.hosts b/services/grafana/.hosts index 8f36ea6..537a644 100644 --- a/services/grafana/.hosts +++ b/services/grafana/.hosts @@ -1 +1,2 @@ IPV4_PREFIX.122 grafana.LOCAL_DOMAIN +IPV4_PREFIX.123 loki.LOCAL_DOMAIN diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 6d27299..90b67f2 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -17,6 +17,14 @@ services: stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] + loki: + image: ${LOKI_IMAGE}:${LOKI_VERSION} + command: -config.file=/etc/loki/local-config.yaml + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.123 + networks: grafana_int: internet: diff --git a/services/grafana/provisioning/dashboards/storage-node-logs.json b/services/grafana/provisioning/dashboards/storage-node-logs.json new file mode 100644 index 0000000..3c91d52 --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node-logs.json @@ -0,0 +1,358 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({hostname=~\"$hostname\"} |~ \"$level\" |~ `$filter` [1s])", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log intensity", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 226 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.width", + "value": 424 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\"} |~ `$filter` | json level |~ \"$level\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "id": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Hostname", + "multi": true, + "name": "hostname", + "options": [], + "query": { + "label": "hostname", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "debug", + "value": "debug" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "warn", + "value": "warn" + }, + { + "selected": false, + "text": "error", + "value": "error" + }, + { + "selected": false, + "text": "dpanic", + "value": "dpanic" + }, + { + "selected": false, + "text": "panic", + "value": "panic" + }, + { + "selected": false, + "text": "fatal", + "value": "fatal" + } + ], + "query": "debug,info,warn,error,dpanic,panic,fatal", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Filter", + "name": "filter", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node logs", + "uid": "bef4b42d-c74e-4241-96cb-d239113f44bc", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml index 81b6a7b..7d7427f 100644 --- a/services/grafana/provisioning/datasources/datasource.yml +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -6,3 +6,8 @@ datasources: access: proxy orgId: 1 url: http://prometheus:9090 +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 \ No newline at end of file diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 0a85096..6b3e7a8 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -1,6 +1,11 @@ # Logger section logger: level: debug # Minimum enabled logging level + loki: + enabled: true + endpoint: "loki.frostfs.devenv:3100/api/prom/push" + max_batch_delay: 1s + max_batch_size: 200 # Profiler section pprof: @@ -14,7 +19,7 @@ prometheus: address: :9090 # Server address shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown - # Application tracing section +# Application tracing section tracing: enabled: true exporter: otlp_grpc From 1077c9d35898ec5dd4ba71436d9f4e6489ab26d3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Dec 2023 14:08:47 +0300 Subject: [PATCH 38/81] [#59] .env: Update neo-go to v0.104.0 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 0969ff5..16d2da5 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.103.0 +NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes From dfad34fdeae470937d797b065ead0e29fe4a01dc Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:50:16 +0300 Subject: [PATCH 39/81] [#61] Update frostfs-node components with contracts Signed-off-by: Alex Vanin --- .env | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 16d2da5..7752b67 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.37.0-rc.1-93-g364f835b +IR_VERSION=0.37.0-rc.1-164-gbdd43f62 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.37.0-rc.1-93-g364f835b +NODE_VERSION=0.37.0-rc.1-164-gbdd43f62 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,17 +40,17 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BbngJDdRJEDJTJk7qptq3SxKqrJqtvVYWU6R5AaFGbtG +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/HUu3vMojqiopmAzWgUPPPSN7ThosapsLj5WrBvZR67tC #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=eca5c210 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/2GxarAjGUb3RevxvqFGYT3hDQxNNaHzK6aFxhJCAMehq +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/NMw4AktoK2zo5Kftj23wGEVzE1d52ULufjtcLjAVqZc #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts FROSTFS_CONTRACTS_VERSION=8537293e -FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/6ccZoj4HxoN1G1qvJAX2Qw9p2D6qdyzAjNMaNkEKYQpA +FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BLG2NrQW5UAF1RSMSBK8Z4mGanhvzD9aTRk4NsRGVDV9 #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing @@ -67,4 +67,4 @@ GRAFANA_IMAGE=grafana/grafana # Loki versions LOKI_VERSION=2.9.1 -LOKI_IMAGE=grafana/loki \ No newline at end of file +LOKI_IMAGE=grafana/loki From 9654b77236aa20f4badcc706a77bb7ed2c930e83 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:53:12 +0300 Subject: [PATCH 40/81] [#61] Update frostfs-http-gw Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 7752b67..99c5ed0 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=0.27.0-rc.1-15-g1776db28 +HTTP_GW_VERSION=0.28.0-rc.1-15-g5ae75eb9 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate From 0f6f2722c2fe672a1246c6f6732c50711d1af271 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:57:17 +0300 Subject: [PATCH 41/81] [#61] Update frostfs-s3-gw Signed-off-by: Alex Vanin --- .env | 2 +- services/s3_gate/cfg/config.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 99c5ed0..793cffc 100644 --- a/.env +++ b/.env @@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.27.0-rc.1-30-gce929468 +S3_GW_VERSION=0.28.0-rc.1-51-g9272f4e1 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 7fa041a..03e84e0 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -33,3 +33,17 @@ server: wallet: path: /wallet.json # Path to wallet passphrase: "s3" # Passphrase to decrypt wallet + +features: + md5: + enabled: true + +control: + grpc: + endpoint: localhost:16515 + +frostfsid: + enabled: false + +policy: + enabled: false From 19e5cec49fdd5fbf244ee234b5bc8cdb81873964 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 20 Feb 2024 15:47:55 +0300 Subject: [PATCH 42/81] [#63] Use SIGTERM to stop Go services Signed-off-by: Evgenii Stratonikov --- services/ir/docker-compose.yml | 3 ++- services/rest_gate/docker-compose.yml | 3 ++- services/s3_gate/docker-compose.yml | 3 ++- services/storage/docker-compose.yml | 12 ++++++++---- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 77ec50d..0f8e0f9 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -13,7 +13,8 @@ services: ir_int: internet: ipv4_address: ${IPV4_PREFIX}.61 - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s volumes: - ./az.json:/wallet.json - ./az.key:/wallet01.key diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml index f5bac6b..ee1ab5e 100644 --- a/services/rest_gate/docker-compose.yml +++ b/services/rest_gate/docker-compose.yml @@ -16,7 +16,8 @@ services: - ./wallet.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/rest - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".int_test.env" ] command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ] environment: diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 1445480..d1b7601 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -18,7 +18,8 @@ services: - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/s3 - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".s3.env", ".int_test.env" ] command: [ "frostfs-s3-gw", "--config", "/etc/frostfs/s3/config.yml" ] environment: diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index b32ba21..1aabc5e 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -24,7 +24,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -64,7 +65,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -104,7 +106,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -145,7 +148,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: From 8edfcb364d9a104e15f976117d20485a864a0d41 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 22 Feb 2024 12:41:55 +0300 Subject: [PATCH 43/81] [#64] adm: Create default Allow policy for root namespace Signed-off-by: Airat Arifullin --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 4caa292..051aa7d 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,10 @@ up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts + echo "Set rule chain to policy contract" + @./vendor/frostfs-adm --config frostfs-adm.yml morph \ + ape add-rule-chain --target-type namespace --target-name "" \ + --rule 'allow Container.* *' --chain-id "allow_container_ops" @for f in ./services/storage/wallet*.json; do \ echo "Transfer GAS to wallet $${f}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ From 47b4917e7bed1eae19b72e2d4dadc521b9b9c870 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 4 Apr 2024 11:38:41 +0300 Subject: [PATCH 44/81] [#65] services: Fix docker-compose warnings There were multiple warning like this one. ``` WARN[0000] /secret/services/rest_gate/docker-compose.yml: `version` is obsolete ``` According to docker-compose spec, the parameter is indeed purely informative: https://github.com/compose-spec/compose-spec/blob/master/spec.md#version-and-name-top-level-elements Signed-off-by: Evgenii Stratonikov --- services/basenet/docker-compose.yml | 1 - services/grafana/docker-compose.yml | 1 - services/http_gate/docker-compose.yml | 1 - services/ir/docker-compose.yml | 1 - services/jaeger/docker-compose.yml | 1 - services/morph_chain/docker-compose.yml | 1 - services/nats/docker-compose.yml | 1 - services/prometheus/docker-compose.yml | 1 - services/rest_gate/docker-compose.yml | 1 - services/s3_gate/docker-compose.yml | 1 - services/storage/docker-compose.yml | 1 - 11 files changed, 11 deletions(-) diff --git a/services/basenet/docker-compose.yml b/services/basenet/docker-compose.yml index abac92a..b2905a3 100644 --- a/services/basenet/docker-compose.yml +++ b/services/basenet/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: basenet: diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 90b67f2..3953400 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: grafana: image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION} diff --git a/services/http_gate/docker-compose.yml b/services/http_gate/docker-compose.yml index 7108ee5..c2e68ae 100644 --- a/services/http_gate/docker-compose.yml +++ b/services/http_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: http_gate: image: ${HTTP_GW_IMAGE}:${HTTP_GW_VERSION} diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 0f8e0f9..048651b 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: ir01: diff --git a/services/jaeger/docker-compose.yml b/services/jaeger/docker-compose.yml index a460548..1fb367b 100644 --- a/services/jaeger/docker-compose.yml +++ b/services/jaeger/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: jaeger: image: ${JAEGER_IMAGE}:${JAEGER_VERSION} diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 4cba4b3..dd44b2c 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: frostfs_morph_chain: image: ${NEOGO_IMAGE}:${NEOGO_VERSION} diff --git a/services/nats/docker-compose.yml b/services/nats/docker-compose.yml index d05dd04..cb89948 100644 --- a/services/nats/docker-compose.yml +++ b/services/nats/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: nats: image: ${NATS_IMAGE}:${NATS_VERSION} diff --git a/services/prometheus/docker-compose.yml b/services/prometheus/docker-compose.yml index e9eb4f9..31f5fbe 100644 --- a/services/prometheus/docker-compose.yml +++ b/services/prometheus/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: prometheus: image: ${PROMETHEUS_IMAGE}:${PROMETHEUS_VERSION} diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml index ee1ab5e..09cfe38 100644 --- a/services/rest_gate/docker-compose.yml +++ b/services/rest_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: rest_gate: image: ${REST_GW_IMAGE}:${REST_GW_VERSION} diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index d1b7601..6b43a0b 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: s3_gate: image: ${S3_GW_IMAGE}:${S3_GW_VERSION} diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 1aabc5e..8a8698e 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: storage01: image: ${NODE_IMAGE}:${NODE_VERSION} From 773ea2339bf1eebc4946a0adbad904d7a8c8debf Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 15 Apr 2024 20:16:51 +0300 Subject: [PATCH 45/81] [#66] Update frostfs-* to v0.38.5 Signed-off-by: Alexander Chuprov --- .env | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 793cffc..ad96218 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.37.0-rc.1-164-gbdd43f62 +IR_VERSION=0.38.5 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.37.0-rc.1-164-gbdd43f62 +NODE_VERSION=0.38.5 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,17 +40,17 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/HUu3vMojqiopmAzWgUPPPSN7ThosapsLj5WrBvZR67tC +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BtA6zWLtoDuDnzxSNkKPjyN5hGuSZFxGuJD1gh8kBwkP #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=eca5c210 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/NMw4AktoK2zo5Kftj23wGEVzE1d52ULufjtcLjAVqZc +FROSTFS_ADM_VERSION=498f9955ea +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/CjV4S6ENzo2FLK4KxXZHHNW4veR1ATtynGY6Mc1xQ6RB #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=8537293e -FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BLG2NrQW5UAF1RSMSBK8Z4mGanhvzD9aTRk4NsRGVDV9 +FROSTFS_CONTRACTS_VERSION=694daebb19 +FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing From 2b6122192a4c26a84e6f8faa5533ab52fb2a88ef Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 8 May 2024 11:33:19 +0300 Subject: [PATCH 46/81] [#67] services/ir: Remove deprecated flag in healthcheck Signed-off-by: Evgenii Stratonikov --- services/ir/healthcheck.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh index 747bc8b..88fcdaa 100755 --- a/services/ir/healthcheck.sh +++ b/services/ir/healthcheck.sh @@ -1,6 +1,6 @@ #!/bin/sh -/frostfs-cli control healthcheck \ +/frostfs-cli control ir healthcheck \ --endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \ - --wallet /wallet01.key --ir | + --wallet /wallet01.key | grep "Health status: READY" From 0be22a937536ae5ef425b338caa676ea0711bbde Mon Sep 17 00:00:00 2001 From: Alexey Savchuk Date: Mon, 8 Jul 2024 10:34:28 +0300 Subject: [PATCH 47/81] [#73] Update HTTP gate docs Signed-off-by: Aleksey Savchuk --- docs/http_gate.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/http_gate.md b/docs/http_gate.md index e2159ad..b6b2591 100644 --- a/docs/http_gate.md +++ b/docs/http_gate.md @@ -22,8 +22,8 @@ Image label prefix to use for containers. - Create a new container ``` $ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ - --key wallets/wallet.key \ - container create --basic-acl readonly --await \ + --wallet wallets/wallet.key \ + container create --basic-acl private --await \ --policy "REP 1 SELECT 1 FROM *" container ID: 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP awaiting... @@ -33,7 +33,7 @@ container has been persisted on sidechain - Put an object into the newly created container ``` $ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ - --key wallets/wallet.key \ + --wallet wallets/wallet.key \ object put --file /tmp/backup.jpeg \ --cid 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP [/tmp/backup.jpeg] Object successfully stored From d03be14312da6b1bd2f5bcbb113c4594eeaff258 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 8 Aug 2024 13:18:13 +0300 Subject: [PATCH 48/81] [#75] Makefile: Add subjects for storage and client wallets to FrostfsID Signed-off-by: Airat Arifullin --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 051aa7d..9483dc7 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,12 @@ up/bootstrap: get vendor/hosts && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ done - @echo "FrostFS sidechain environment is deployed" + @echo "Create frostfsid subject for ./wallets/wallet.json" \ + && subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ + && echo "Subject key: $${subj_key}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \ + || die "Failed to create subject for the wallet"; \ + echo "FrostFS sidechain environment is deployed" # Build up certain service .PHONY: up/% From f94fa284ece87feb5c6dd9bf18b664da916b2656 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 05:50:27 +0300 Subject: [PATCH 49/81] [#76] Update frostfs-core components to v0.42.9 Signed-off-by: Alexander Chuprov --- .env | 12 ++++++------ services/ir/artifacts.mk | 7 ++----- services/morph_chain/artifacts.mk | 7 ++----- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.env b/.env index ad96218..b414b0d 100644 --- a/.env +++ b/.env @@ -12,12 +12,12 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.38.5 -IR_IMAGE=truecloudlab/frostfs-ir +IR_VERSION=v0.42.9 +IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.38.5 -NODE_IMAGE=truecloudlab/frostfs-storage +NODE_VERSION=v0.42.9 +NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # NATS Server NATS_VERSION=2.7.2 @@ -40,12 +40,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BtA6zWLtoDuDnzxSNkKPjyN5hGuSZFxGuJD1gh8kBwkP +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}//frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=498f9955ea -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/CjV4S6ENzo2FLK4KxXZHHNW4veR1ATtynGY6Mc1xQ6RB +FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index 95b2c22..0cdbdbb 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -25,7 +25,6 @@ endif # Download FrostFS CLI .ONESHELL: get.cli: FROSTFS_CLI_FILE=./vendor/frostfs-cli -get.cli: FROSTFS_CLI_ARCHIVE_FILE=${FROSTFS_CLI_FILE}.tar.gz get.cli: FROSTFS_CLI_PATH?= get.cli: @mkdir -p ./vendor @@ -34,10 +33,8 @@ ifeq (${FROSTFS_CLI_PATH},) @echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}" @curl \ -ksSL "${FROSTFS_CLI_URL}" \ - -o ${FROSTFS_CLI_ARCHIVE_FILE} - @tar -xvf ${FROSTFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${FROSTFS_CLI_FILE} - @rm ${FROSTFS_CLI_ARCHIVE_FILE} + -o ${FROSTFS_CLI_FILE} + @chmod +x ${FROSTFS_CLI_FILE} else @echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}" @cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE} diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 3572e69..77a7ae3 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -20,15 +20,12 @@ endif # Download FrostFS ADM tool get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm -get.adm: FROSTFS_ADM_ARCHIVE=frostfs-adm.tar.gz get.adm: ifeq (${FROSTFS_ADM_PATH},) @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}" - @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_ARCHIVE} - @tar -xvf ${FROSTFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${FROSTFS_ADM_DEST} - @rm ${FROSTFS_ADM_ARCHIVE} + @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST} + @chmod +x ${FROSTFS_ADM_DEST} else @echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}" @cp ${FROSTFS_ADM_PATH} ${FROSTFS_ADM_DEST} From 155042343b40c94507af61d0cbc7598dfed995a0 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 06:25:11 +0300 Subject: [PATCH 50/81] [#69] service/storage: Add support -q flag in healthcheck Signed-off-by: Alexander Chuprov --- services/storage/docker-compose.yml | 12 ++++-------- services/storage/healthcheck.sh | 5 ----- 2 files changed, 4 insertions(+), 13 deletions(-) delete mode 100755 services/storage/healthcheck.sh diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 8a8698e..fb10974 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -17,7 +17,6 @@ services: - storage_s01:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -36,7 +35,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW - FROSTFS_NODE_ATTRIBUTE_2=Price:22 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -58,7 +57,6 @@ services: - storage_s02:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -77,7 +75,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED - FROSTFS_NODE_ATTRIBUTE_2=Price:33 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -99,7 +97,6 @@ services: - storage_s03:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -118,7 +115,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO - FROSTFS_NODE_ATTRIBUTE_2=Price:11 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -140,7 +137,6 @@ services: - storage_s04:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/tls.crt - ./s04tls.key:/tls.key - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert @@ -165,7 +161,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL - FROSTFS_NODE_ATTRIBUTE_2=Price:44 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 diff --git a/services/storage/healthcheck.sh b/services/storage/healthcheck.sh deleted file mode 100755 index 792d2ad..0000000 --- a/services/storage/healthcheck.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -/frostfs-cli control healthcheck -c /cli-cfg.yml \ - --endpoint "$FROSTFS_CONTROL_GRPC_ENDPOINT" | - grep "Health status: READY" From ae658469a5d44191e51df539054b3f0367bfdb73 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 06:25:24 +0300 Subject: [PATCH 51/81] [#69] service/ir: Add support -q flag in healthcheck Signed-off-by: Alexander Chuprov --- services/ir/docker-compose.yml | 3 +-- services/ir/healthcheck.sh | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100755 services/ir/healthcheck.sh diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 048651b..4e671b2 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -20,12 +20,11 @@ services: - ./../../vendor/hosts:/etc/hosts - ./../../vendor/locode_db:/locode/db - ./../../vendor/frostfs-cli:/frostfs-cli - - ./healthcheck.sh:/healthcheck.sh - ./cfg:/etc/frostfs/ir env_file: [ ".env", ".ir.env", ".int_test.env" ] command: [ "frostfs-ir", "--config", "/etc/frostfs/ir/config.yml" ] healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control ir healthcheck -q --wallet /wallet01.key --endpoint \"$$FROSTFS_IR_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh deleted file mode 100755 index 88fcdaa..0000000 --- a/services/ir/healthcheck.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -/frostfs-cli control ir healthcheck \ - --endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \ - --wallet /wallet01.key | - grep "Health status: READY" From 044cf99e8dae647abb3706b498b27d00b58b6edc Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 08:08:24 +0300 Subject: [PATCH 52/81] [#70] Makefile: Make bootstrap idempotent Signed-off-by: Alexander Chuprov --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9483dc7..2f505a9 100644 --- a/Makefile +++ b/Makefile @@ -85,11 +85,15 @@ up/bootstrap: get vendor/hosts && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ done - @echo "Create frostfsid subject for ./wallets/wallet.json" \ - && subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ + @echo "Create frostfsid subject for ./wallets/wallet.json"; \ + if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ + echo "Subject already exists"; \ + else \ + subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ && echo "Subject key: $${subj_key}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \ || die "Failed to create subject for the wallet"; \ + fi echo "FrostFS sidechain environment is deployed" # Build up certain service From dd382f8ce048d9e0880f43471d7896de000c0843 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 08:08:55 +0300 Subject: [PATCH 53/81] [#68] service/morph: Add volume for morph_chain Signed-off-by: Alexander Chuprov --- services/morph_chain/docker-compose.yml | 4 ++++ services/morph_chain/protocol.privnet.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index dd44b2c..68535ea 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,9 +19,13 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - chains:/chains networks: chain_int: internet: external: true name: basenet_internet + +volumes: + chains: diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 560730a..2066029 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -17,7 +17,7 @@ ApplicationConfiguration: DBConfiguration: Type: "boltdb" BoltDBOptions: - FilePath: "./db/morph.bolt" + FilePath: "/chains/morph.bolt" P2P: Addresses: - ":20333" From 439a9e71cfaf3837f16d9c501385c8ed2ff67dbc Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 6 Sep 2024 14:09:08 +0300 Subject: [PATCH 54/81] [#81] env: Fix typo Signed-off-by: Alexander Chuprov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index b414b0d..c62cf17 100644 --- a/.env +++ b/.env @@ -40,7 +40,7 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}//frostfs-cli +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary From 2e67acbcb2af952933b87856083a6a926b840723 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 6 Sep 2024 14:37:48 +0300 Subject: [PATCH 55/81] [#78] env: Restore version format Signed-off-by: Alexander Chuprov --- .env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index c62cf17..dea5726 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=v0.42.9 +IR_VERSION=0.42.9 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=v0.42.9 +NODE_VERSION=0.42.9 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # NATS Server @@ -40,12 +40,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-cli +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=498f9955ea -FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-adm +FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts From 7538bd9b171409fc5eeb5c08d074efe5792db133 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Sep 2024 16:45:06 +0300 Subject: [PATCH 56/81] [#83] Honor IPV4_PREFIX in morph_chain config Signed-off-by: Vitaliy Potyarkin --- services/morph_chain/protocol.privnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 2066029..f1cc754 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -36,7 +36,7 @@ ApplicationConfiguration: Password: "one" RPC: Addresses: - - "192.168.130.90:30333" + - ":30333" Enabled: true SessionEnabled: true EnableCORSWorkaround: false From d0c32731f2c468dafd979337ad04af1075d2bca2 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 9 Sep 2024 18:59:33 +0300 Subject: [PATCH 57/81] [#80] Update frostfs-service components to v0.30.* Signed-off-by: Alexander Chuprov --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index dea5726..17697f6 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=0.28.0-rc.1-15-g5ae75eb9 +HTTP_GW_VERSION=0.30.2 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate @@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.28.0-rc.1-51-g9272f4e1 +S3_GW_VERSION=0.30.4 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database From 636be7352ebf5221ed66b02a8df85cead7c3633e Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 30 Sep 2024 17:04:11 +0300 Subject: [PATCH 58/81] [#84] Make targets for issuing credentials Signed-off-by: Nikita Zinkevich --- Makefile | 1 + README.md | 59 +++++++++++++++++++++++++ services/morph_chain/docker-compose.yml | 1 + services/s3_gate/docker-compose.yml | 8 ++++ services/s3_gate/issue-creds.sh | 41 +++++++++++++++++ services/s3_gate/prepare.mk | 14 ++++++ 6 files changed, 124 insertions(+) create mode 100755 services/s3_gate/issue-creds.sh create mode 100644 services/s3_gate/prepare.mk diff --git a/Makefile b/Makefile index 2f505a9..dff81df 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,7 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) .PHONY: up up: up/basic @$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) + ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet" @echo "Full FrostFS Developer Environment is ready" # Build up FrostFS diff --git a/README.md b/README.md index aa07a08..ad57655 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,65 @@ Display addresses and host names for each running service, if available. Clean up `vendor` directory. +### s3cred + +Registers user wallet and issues s3 credentials. + +Usage and default parameter values: +```sh +make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] +``` + +As soon as the storage node is in the network map (see above) you can generate S3 +credentials: + +``` sh +$ make s3cred +{ + "access_key_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p", + "secret_access_key": "d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f", + "owner_private_key": "140947599afd9ca89af4b358c3176eb046e554d942a0dc99a8e06f3e43c8f4ad", + "wallet_public_key": "0324e76288fcb900100d01802a14ef977cca45ad073561230446df14b344c858b6", + "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT" +} +``` +Running without any parameters will result in defaults which are based on the private key from +`/user-wallet.json` file and `/wallet.json` contract wallet. + +Now let's configure an S3 client (AWS CLI will be used as example): + +``` sh +$ aws configure +AWS Access Key ID []: EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p +AWS Secret Access Key []: d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f +Default region name []: us-east-1 +Default output format []: json +``` + +If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter. +Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params. + +```sh +$ make s3cred wallet=custom_wallet.json password=test +{ + "access_key_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys904W7xXFNKqem2ACvTRWRYJsZMCUikYFSokN7pPJziWyDi", + "secret_access_key": "21bb64fafa32c82417fd8b97ac56cc8a085998a3852632d52fe7042453daa440", + "owner_private_key": "10f6f9d7a47bb0bf68363ad8a99fe69f1493f8b6e1665b3e4e83feb2d5c7ee39", + "wallet_public_key": "03e38759973a6bb722baabc2dd84036a39f0b2f53d32fec45a4dacde8a50fe4b70", + "container_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys9" +} +``` + +To get credentials from custom wallet, place it in `wallets` dir before start. + +### cred + +Usage and default parameter values: +```sh +make cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] +``` +The same as `s3cred`, but it doesn't issues s3 credentials. + ## Contributing Feel free to contribute to this project after reading the [contributing diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 68535ea..2c2facb 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,6 +19,7 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - ./../s3_gate/wallet.json:/wallets/s3-wallet.json - chains:/chains networks: diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 6b43a0b..f7fc28d 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -12,11 +12,17 @@ services: internet: ipv4_address: ${IPV4_PREFIX}.82 volumes: + # Gate wallet - ./wallet.json:/wallet.json + # Custom user wallets + - ./wallets:/wallets + # Default user wallet + - ./../../wallets/wallet.json:/wallets/wallet.json - ./tls.key:/tls.key - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/s3 + - ./issue-creds.sh:/usr/bin/issue-creds.sh stop_signal: SIGTERM stop_grace_period: 15s env_file: [ ".env", ".s3.env", ".int_test.env" ] @@ -34,6 +40,8 @@ services: - S3_GW_PEERS_2_WEIGHT=0.2 - S3_GW_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_3_WEIGHT=0.2 + - AUTHMATE_WALLET_PASSPHRASE= + - AUTHMATE_WALLET_CONTRACT_PASSPHRASE=s3 networks: s3_gate_int: diff --git a/services/s3_gate/issue-creds.sh b/services/s3_gate/issue-creds.sh new file mode 100755 index 0000000..e2355f8 --- /dev/null +++ b/services/s3_gate/issue-creds.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +initUser() { + /bin/frostfs-s3-authmate register-user \ + --wallet $WALLET_PATH \ + --rpc-endpoint http://morph-chain.frostfs.devenv:30333 \ + --username $USERNAME \ + --contract-wallet /wallet.json 1> /dev/null && touch $WALLET_CACHE/$USERNAME +} + +issueCreds() { + /bin/frostfs-s3-authmate issue-secret \ + --wallet $WALLET_PATH \ + --peer s01.frostfs.devenv:8080 \ + --gate-public-key $S3_GATE_PUBLIC_KEY \ + --container-placement-policy "REP 3" +} + +set -e + +WALLET_PATH=/wallets/$2 +if [[ -z "$2" ]]; then + WALLET_PATH=/wallets/wallet.json +fi + +S3_GATE_PUBLIC_KEY=$3 +if [[ -z "$3" ]]; then + S3_GATE_PUBLIC_KEY=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf +fi + +WALLET_CACHE=/data/wallets +mkdir -p $WALLET_CACHE + +USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1) +if [ ! -e $WALLET_CACHE/$USERNAME ]; then + initUser +fi + +if [ $1 == "s3" ]; then + issueCreds +fi diff --git a/services/s3_gate/prepare.mk b/services/s3_gate/prepare.mk new file mode 100644 index 0000000..c97add8 --- /dev/null +++ b/services/s3_gate/prepare.mk @@ -0,0 +1,14 @@ +.PHONY: s3cred register + +password?= +contract_password?=s3 +gate_public_key?= +wallet?= + +# Register wallet & generate S3 credentials +s3cred: + @docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh s3 "$(wallet)" "$(gate_public_key)" + +# Only registers user wallet +register: + @docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh native "$(wallet)" From 7152f59232ac5ac763cbb6433331d161f64c56d0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Oct 2024 10:24:30 +0300 Subject: [PATCH 59/81] [#88] Remove nats service It was removed from node in TrueCloudLab/frostfs-node#1161. Signed-off-by: Evgenii Stratonikov --- .basic_services | 1 - .env | 4 --- .gitignore | 1 - Makefile | 2 +- services/nats/.env | 1 - services/nats/.hosts | 1 - services/nats/.int_test.env | 1 - services/nats/artifacts.mk | 7 ----- services/nats/docker-compose.yml | 30 ------------------ services/nats/generate_cert.sh | 49 ----------------------------- services/nats/nats.conf | 15 --------- services/storage/cfg/config.yml | 8 ----- services/storage/docker-compose.yml | 12 ------- 13 files changed, 1 insertion(+), 131 deletions(-) delete mode 120000 services/nats/.env delete mode 100644 services/nats/.hosts delete mode 120000 services/nats/.int_test.env delete mode 100644 services/nats/artifacts.mk delete mode 100644 services/nats/docker-compose.yml delete mode 100755 services/nats/generate_cert.sh delete mode 100644 services/nats/nats.conf diff --git a/.basic_services b/.basic_services index 201ea70..845c731 100644 --- a/.basic_services +++ b/.basic_services @@ -1,5 +1,4 @@ # Services start/stop order # Will start from top to bottom and stop in reverse -nats ir storage diff --git a/.env b/.env index 17697f6..6bb3728 100644 --- a/.env +++ b/.env @@ -19,10 +19,6 @@ IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir NODE_VERSION=0.42.9 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage -# NATS Server -NATS_VERSION=2.7.2 -NATS_IMAGE=nats - # HTTP Gate HTTP_GW_VERSION=0.30.2 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw diff --git a/.gitignore b/.gitignore index 36e495d..a672692 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,3 @@ sites/* # Runtime generation keys services/storage/*tls.crt services/storage/*tls.key -services/nats/*.pem diff --git a/Makefile b/Makefile index dff81df..c4a6c48 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ hosts: vendor/hosts .PHONY: clean .ONESHELL: clean: - @rm -rf vendor/* services/storage/s04tls.* services/nats/*.pem + @rm -rf vendor/* services/storage/s04tls.* @> .int_test.env @for svc in $(PULL_SVCS) do diff --git a/services/nats/.env b/services/nats/.env deleted file mode 120000 index c7360fb..0000000 --- a/services/nats/.env +++ /dev/null @@ -1 +0,0 @@ -../../.env \ No newline at end of file diff --git a/services/nats/.hosts b/services/nats/.hosts deleted file mode 100644 index 6ed5417..0000000 --- a/services/nats/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.101 nats.LOCAL_DOMAIN diff --git a/services/nats/.int_test.env b/services/nats/.int_test.env deleted file mode 120000 index 582b6a2..0000000 --- a/services/nats/.int_test.env +++ /dev/null @@ -1 +0,0 @@ -../../.int_test.env \ No newline at end of file diff --git a/services/nats/artifacts.mk b/services/nats/artifacts.mk deleted file mode 100644 index b66cebf..0000000 --- a/services/nats/artifacts.mk +++ /dev/null @@ -1,7 +0,0 @@ -# Create new TLS certs for NATS server and clients - -NATS_DIR=$(abspath services/nats) - -get.nats: - @echo "⇒ Creating certs for NATS server and clients" - ${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null diff --git a/services/nats/docker-compose.yml b/services/nats/docker-compose.yml deleted file mode 100644 index cb89948..0000000 --- a/services/nats/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -services: - nats: - image: ${NATS_IMAGE}:${NATS_VERSION} - domainname: ${LOCAL_DOMAIN} - hostname: nats - container_name: nats - restart: on-failure - dns: - - ${IPV4_PREFIX}.101 - networks: - nats_int: - internet: - ipv4_address: ${IPV4_PREFIX}.101 - volumes: - - ./../../vendor/hosts:/etc/hosts - - ./nats.conf:/etc/nats/frostfs-nats-server.conf - - ./server-cert.pem:/certs/server-cert.pem - - ./server-key.pem:/certs/server-key.pem - - ./ca-cert.pem:/certs/ca-cert.pem - stop_signal: SIGKILL - env_file: [ ".env", ".int_test.env" ] - command: ["-c", "/etc/nats/frostfs-nats-server.conf"] - -networks: - nats_int: - internet: - external: true - name: basenet_internet diff --git a/services/nats/generate_cert.sh b/services/nats/generate_cert.sh deleted file mode 100755 index 4c4dd0b..0000000 --- a/services/nats/generate_cert.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -source bin/helper.sh - -WORKDIR=$(dirname "$0") -LOCAL_DOMAIN=$1 - -CA_KEY=$WORKDIR/ca-key.pem -CA_CRT=$WORKDIR/ca-cert.pem - -SRV_KEY=$WORKDIR/server-key.pem -SRV_REQ=$WORKDIR/server-req.csr -SRV_CRT=$WORKDIR/server-cert.pem - -CLI_KEY=$WORKDIR/client-key.pem -CLI_REQ=$WORKDIR/client-req.csr -CLI_CRT=$WORKDIR/client-cert.pem - -SUBJ="/O=TrueCloudLab" - -if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then - openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 || - die "CA certificate was not created" -fi - -if [[ ! -f $SRV_KEY || ! -f $SRV_CRT ]]; then - openssl req -newkey rsa:4096 -nodes -keyout $SRV_KEY -out $SRV_REQ -subj $SUBJ 2>&1 || - die "Server certificate was not created" - - openssl x509 -req -days 365 -set_serial 01 -in $SRV_REQ -out $SRV_CRT -CA $CA_CRT -CAkey $CA_KEY \ - -extensions san -extfile <(printf "[san]\nsubjectAltName=DNS:nats.$LOCAL_DOMAIN") 2>&1 || { - rm $SRV_REQ - die "Server certificate was not signed by CA" - } - - rm $SRV_REQ -fi - -if [[ ! -f $CLI_KEY || ! -f $CLI_CRT ]]; then - openssl req -newkey rsa:4096 -nodes -keyout $CLI_KEY -out $CLI_REQ -subj $SUBJ 2>&1 || - die "Client certificate was not created" - - openssl x509 -req -days 365 -set_serial 01 -in $CLI_REQ -out $CLI_CRT -CA $CA_CRT -CAkey $CA_KEY 2>&1 || { - rm $CLI_REQ - die "Client certificate was not signed by CA" - } - - rm $CLI_REQ -fi diff --git a/services/nats/nats.conf b/services/nats/nats.conf deleted file mode 100644 index 0c7af0a..0000000 --- a/services/nats/nats.conf +++ /dev/null @@ -1,15 +0,0 @@ -port: 4222 -monitor_port: 8222 - -jetstream { - store_dir=nats - max_memory_store: 1GB - max_file_store: 2GB -} - -tls { - cert_file: /certs/server-cert.pem - key_file: /certs/server-key.pem - ca_file: /certs/ca-cert.pem - verify: true -} diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 6b3e7a8..8672069 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -35,14 +35,6 @@ morph: # Common storage node settings node: attribute_0: "User-Agent:FrostFS/0.34" - notification: - enabled: true # Turn on object notification service - endpoint: "tls://nats.frostfs.devenv:4222" # Notification server endpoint - timeout: "6s" # Timeout for object notification client connection - default_topic: "test" # Default topic for object notifications if not found in object's meta - certificate: "/etc/frostfs-node/nats.tls.cert" # Path to TLS certificate - key: "/etc/frostfs-node/nats.tls.key" # Path to TLS key - ca: "/etc/frostfs-node/nats.ca.crt" # Path to optional CA certificate # Tree section tree: diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index fb10974..6a070be 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -18,9 +18,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -58,9 +55,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -98,9 +92,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -139,9 +130,6 @@ services: - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/tls.crt - ./s04tls.key:/tls.key - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s From 10e5bed2afec13cd9803f012360c1bd7dcd184c7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Oct 2024 10:38:09 +0300 Subject: [PATCH 60/81] [#79] storage: Take User-Agent from NODE_VERSION Signed-off-by: Evgenii Stratonikov --- services/storage/cfg/config.yml | 4 ---- services/storage/docker-compose.yml | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 8672069..0e2526d 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -32,10 +32,6 @@ morph: - address: ws://morph-chain:30333/ws priority: 1 -# Common storage node settings -node: - attribute_0: "User-Agent:FrostFS/0.34" - # Tree section tree: enabled: true diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 6a070be..785ce79 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -29,6 +29,7 @@ services: - FROSTFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW - FROSTFS_NODE_ATTRIBUTE_2=Price:22 healthcheck: @@ -66,6 +67,7 @@ services: - FROSTFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED - FROSTFS_NODE_ATTRIBUTE_2=Price:33 healthcheck: @@ -103,6 +105,7 @@ services: - FROSTFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO - FROSTFS_NODE_ATTRIBUTE_2=Price:11 healthcheck: @@ -146,6 +149,7 @@ services: - FROSTFS_GRPC_1_TLS_ENABLED=true - FROSTFS_GRPC_1_TLS_CERTIFICATE=/tls.crt - FROSTFS_GRPC_1_TLS_KEY=/tls.key + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL - FROSTFS_NODE_ATTRIBUTE_2=Price:44 healthcheck: From a0fdaebbf41d847badeac4e39961ef4cf95df3d0 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Fri, 25 Oct 2024 16:24:26 +0300 Subject: [PATCH 61/81] [#85] Add s3 lifecycler Signed-off-by: Pavel Pogodaev --- .env | 6 +++- .services | 1 + services/s3_lifecycler/.env | 1 + services/s3_lifecycler/.hosts | 1 + services/s3_lifecycler/.int_test.env | 1 + services/s3_lifecycler/cfg/config.yml | 42 +++++++++++++++++++++++ services/s3_lifecycler/docker-compose.yml | 38 ++++++++++++++++++++ services/s3_lifecycler/wallet.json | 30 ++++++++++++++++ 8 files changed, 119 insertions(+), 1 deletion(-) create mode 120000 services/s3_lifecycler/.env create mode 100644 services/s3_lifecycler/.hosts create mode 120000 services/s3_lifecycler/.int_test.env create mode 100644 services/s3_lifecycler/cfg/config.yml create mode 100644 services/s3_lifecycler/docker-compose.yml create mode 100644 services/s3_lifecycler/wallet.json diff --git a/.env b/.env index 6bb3728..a1a1586 100644 --- a/.env +++ b/.env @@ -28,9 +28,13 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.30.4 +S3_GW_VERSION=0.31.0-rc.4 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw +# Lifecycler +S3_LIFECYCLER_VERSION=0.1.3 +S3_LIFECYCLER_IMAGE=truecloudlab/frostfs-s3-lifecycler + # FrostFS LOCODE database LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54 #LOCODE_DB_PATH=/path/to/locode_db diff --git a/.services b/.services index d9f34c7..d165241 100644 --- a/.services +++ b/.services @@ -3,3 +3,4 @@ http_gate s3_gate rest_gate +s3_lifecycler diff --git a/services/s3_lifecycler/.env b/services/s3_lifecycler/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/s3_lifecycler/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/s3_lifecycler/.hosts b/services/s3_lifecycler/.hosts new file mode 100644 index 0000000..8eb1796 --- /dev/null +++ b/services/s3_lifecycler/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.84 lifecycler.LOCAL_DOMAIN diff --git a/services/s3_lifecycler/.int_test.env b/services/s3_lifecycler/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/s3_lifecycler/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/s3_lifecycler/cfg/config.yml b/services/s3_lifecycler/cfg/config.yml new file mode 100644 index 0000000..2555980 --- /dev/null +++ b/services/s3_lifecycler/cfg/config.yml @@ -0,0 +1,42 @@ +logger: + level: debug + +prometheus: + enabled: true + address: :9090 + +lifecycle: + job_fetcher_buffer: 1000 + executor_pool_size: 100 + +frostfs: + stream_timeout: 10s + connect_timeout: 10s + healthcheck_timeout: 15s + rebalance_interval: 60s + pool_error_threshold: 100 + tree_pool_max_attempts: 4 + +credential: + use: wallets + source: + wallets: + - path: /wallet.json + address: NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7 + passphrase: "cycle" + - path: /user-wallet.json + address: NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM + passphrase: "" + +morph: + reconnect_clients_interval: 30s + dial_timeout: 5s + contract: + netmap: netmap.frostfs + frostfsid: frostfsid.frostfs + container: container.frostfs + +# Wallet configuration +wallet: + path: /wallet.json # Path to wallet + passphrase: "cycle" # Passphrase to decrypt wallet diff --git a/services/s3_lifecycler/docker-compose.yml b/services/s3_lifecycler/docker-compose.yml new file mode 100644 index 0000000..3456d0c --- /dev/null +++ b/services/s3_lifecycler/docker-compose.yml @@ -0,0 +1,38 @@ +--- + +version: "2.4" +services: + s3_lifecycler: + image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: s3_lifecycler + container_name: s3_lifecycler + restart: on-failure + networks: + s3_lifecycler_int: + internet: + ipv4_address: ${IPV4_PREFIX}.84 + volumes: + - ./wallet.json:/wallet.json + - ./../../vendor/hosts:/etc/hosts + - ./cfg:/etc/frostfs/s3-lifecycler + - ./../../wallets/wallet.json:/user-wallet.json + stop_signal: SIGKILL + env_file: [ ".env", ".int_test.env" ] + command: [ "frostfs-s3-lifecycler", "--config", "/etc/frostfs/s3-lifecycler/config.yml" ] + environment: + - S3_LIFECYCLER_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws + - S3_LIFECYCLER_FROSTFS_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_0_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_1_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_2_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_3_WEIGHT=0.2 + +networks: + s3_lifecycler_int: + internet: + external: true + name: basenet_internet diff --git a/services/s3_lifecycler/wallet.json b/services/s3_lifecycler/wallet.json new file mode 100644 index 0000000..ff8f34e --- /dev/null +++ b/services/s3_lifecycler/wallet.json @@ -0,0 +1,30 @@ +{ + "version": "1.0", + "accounts": [ + { + "address": "NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7", + "key": "6PYR3XurAyTzVeDG5WV2Z8vnGdySw3mTLuKjr6Nwo7tae64SJ7XjZSMMPQ", + "label": "lifecycler", + "contract": { + "script": "DCED9z0M+WSGfXZGxYLj1yYwmgxJXE/kNA4+oWNi0q1uKCdBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "isDefault": false + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} From 326578f0ab2ead1423e137b012ed7dd0e3707f33 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 6 Nov 2024 11:31:27 +0300 Subject: [PATCH 62/81] [#90] Stop using obsolete .github directory This commit is a part of multi-repo cleanup effort: https://git.frostfs.info/TrueCloudLab/frostfs-infra/issues/136 Signed-off-by: Vitaliy Potyarkin --- .dockerignore | 1 - {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md | 0 {.github => .forgejo}/ISSUE_TEMPLATE/config.yml | 0 {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md | 0 {.github => .forgejo}/logo.svg | 0 .github/CODEOWNERS | 1 - CODEOWNERS | 1 + README.md | 2 +- 8 files changed, 2 insertions(+), 3 deletions(-) rename {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/config.yml (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md (100%) rename {.github => .forgejo}/logo.svg (100%) delete mode 100644 .github/CODEOWNERS create mode 100644 CODEOWNERS diff --git a/.dockerignore b/.dockerignore index 2f9d4b4..3867906 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ .docker -.github .forgejo vendor tmp diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.forgejo/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .forgejo/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.forgejo/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/config.yml rename to .forgejo/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.forgejo/ISSUE_TEMPLATE/feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .forgejo/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/logo.svg b/.forgejo/logo.svg similarity index 100% rename from .github/logo.svg rename to .forgejo/logo.svg diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index a4413b0..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @alexvanin @fyrchik diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..d28053e --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +.* @alexvanin @fyrchik diff --git a/README.md b/README.md index ad57655..8e4b714 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-FrostFS logo +FrostFS logo

FrostFS local Development and Testing environment From df6859d2463d768691d0d65d85961748e2309802 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 13 Dec 2024 18:10:57 +0300 Subject: [PATCH 63/81] [#92] docs: Update contract list Contract list got outdated: it was mentioning NeoFS, Audit and Reputation Signed-off-by: Vitaliy Potyarkin --- docs/morph.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/morph.md b/docs/morph.md index 56edfbc..d713bf9 100644 --- a/docs/morph.md +++ b/docs/morph.md @@ -4,14 +4,16 @@ A single-node N3 privnet deployment, running on Contracts deployed: - Alphabet (AZ) [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/alphabet) -- Audit [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/audit) - Balance [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/balance) - Container [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/container) +- FrostFS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfs) +- FrostFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfsid) +- NNS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/nns) - Netmap [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/netmap) -- NeoFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/neofsid) +- Policy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/policy) +- Processing [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/processing) - Proxy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/proxy) -- Reputation [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/reputation) - + RPC available at `http://morph-chain.frostfs.devenv:30333`. ## .env settings From 0f9000bce6aedc9f1f376275ced6bc6a38cf6994 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Dec 2024 15:42:13 +0300 Subject: [PATCH 64/81] [#91] Refine CODEOWNERS settings Signed-off-by: Vitaliy Potyarkin --- CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index d28053e..3b080ce 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,3 @@ -.* @alexvanin @fyrchik +.* @alexvanin @fyrchik +.forgejo/.* @potyarkin +Makefile @potyarkin From 98484b97396d71d5cad90cc4ba504a743e9ba843 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 14:58:00 +0300 Subject: [PATCH 65/81] [#93] Update neo-go to the latest version used by frostfs-node Signed-off-by: Alex Vanin --- .env | 2 +- services/morph_chain/protocol.privnet.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index a1a1586..4727fb1 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.104.0 +NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index f1cc754..7cea631 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -11,6 +11,7 @@ ProtocolConfiguration: VerifyTransactions: true StateRootInHeader: true P2PSigExtensions: true + Hardforks: {} ApplicationConfiguration: SkipBlockVerification: false From e1b8fe791902f87b1347b3f2e948e4fc0fc583f1 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 15:17:21 +0300 Subject: [PATCH 66/81] [#93] Update frostfs-node to the latest version Signed-off-by: Alex Vanin --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 4727fb1..a94341c 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.42.9 +IR_VERSION=0.44.4 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.42.9 +NODE_VERSION=0.44.4 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate From 4db8ca356d5ebe8490ddfad69892da46b80e0ab6 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 15:17:54 +0300 Subject: [PATCH 67/81] [#93] Remove unused env variables These variables were used when binaries were available in public storage network Signed-off-by: Alex Vanin --- .env | 2 -- 1 file changed, 2 deletions(-) diff --git a/.env b/.env index a94341c..aa96d95 100644 --- a/.env +++ b/.env @@ -44,12 +44,10 @@ FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/down #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=498f9955ea FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=694daebb19 FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir From 6a5817e15c1031b1aa06ebcb77dd8086be3079ee Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:08:25 +0300 Subject: [PATCH 68/81] [#93] Register storage nodes in proxy contract during bootstrap This allows to send APE requests to the nodes right after `make up` command Signed-off-by: Alex Vanin --- Makefile | 2 ++ services/morph_chain/docker-compose.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Makefile b/Makefile index c4a6c48..58a12e0 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,8 @@ up/bootstrap: get vendor/hosts echo "Transfer GAS to wallet $${f}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ + echo "Register storage wallet $${f} in proxy contract" \ + && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$${f##*/} | head -1 | awk '{print $1}'` || die "Couldn't set storage allet as proxy wallet" done @echo "Create frostfsid subject for ./wallets/wallet.json"; \ if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 2c2facb..6b83716 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -20,6 +20,10 @@ services: - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json - ./../s3_gate/wallet.json:/wallets/s3-wallet.json + - ./../storage/wallet01.json:/wallets/storage/wallet01.json + - ./../storage/wallet02.json:/wallets/storage/wallet02.json + - ./../storage/wallet03.json:/wallets/storage/wallet03.json + - ./../storage/wallet04.json:/wallets/storage/wallet04.json - chains:/chains networks: From d29d50a002c672a9ddfc6f6950dcba97e45344bc Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:12:06 +0300 Subject: [PATCH 69/81] [#93] Update frostfs-contract to the latest version Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index aa96d95..fe484c6 100644 --- a/.env +++ b/.env @@ -48,7 +48,7 @@ FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/down #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz +FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.21.1/frostfs-contract-v0.21.1.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing From b08bb663f93b0aca97905e6bf8d4796cd8570328 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:28:55 +0300 Subject: [PATCH 70/81] [#93] Update gateway components Signed-off-by: Alex Vanin --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index fe484c6..7a58e35 100644 --- a/.env +++ b/.env @@ -20,7 +20,7 @@ NODE_VERSION=0.44.4 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.30.2 +HTTP_GW_VERSION=0.32.0 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate @@ -28,7 +28,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.31.0-rc.4 +S3_GW_VERSION=0.32.0 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # Lifecycler From d34d842700821a4eb8bcfb650734acb196957c64 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:38:39 +0300 Subject: [PATCH 71/81] [#93] Remove frostfs-rest-gw from dev-env This repo is being archived and not maintained at the moment. Signed-off-by: Alex Vanin --- .env | 4 ---- .services | 1 - services/rest_gate/.env | 1 - services/rest_gate/.hosts | 1 - services/rest_gate/.int_test.env | 1 - services/rest_gate/cfg/config.yml | 12 ---------- services/rest_gate/docker-compose.yml | 32 --------------------------- services/rest_gate/wallet.json | 30 ------------------------- 8 files changed, 82 deletions(-) delete mode 120000 services/rest_gate/.env delete mode 100644 services/rest_gate/.hosts delete mode 120000 services/rest_gate/.int_test.env delete mode 100644 services/rest_gate/cfg/config.yml delete mode 100644 services/rest_gate/docker-compose.yml delete mode 100644 services/rest_gate/wallet.json diff --git a/.env b/.env index 7a58e35..824a4db 100644 --- a/.env +++ b/.env @@ -23,10 +23,6 @@ NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage HTTP_GW_VERSION=0.32.0 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw -# REST Gate -REST_GW_VERSION=c9c85e90 -REST_GW_IMAGE=truecloudlab/frostfs-rest-gw - # S3 Gate S3_GW_VERSION=0.32.0 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw diff --git a/.services b/.services index d165241..ccc96f8 100644 --- a/.services +++ b/.services @@ -2,5 +2,4 @@ # Will start from top to bottom and stop in reverse http_gate s3_gate -rest_gate s3_lifecycler diff --git a/services/rest_gate/.env b/services/rest_gate/.env deleted file mode 120000 index c7360fb..0000000 --- a/services/rest_gate/.env +++ /dev/null @@ -1 +0,0 @@ -../../.env \ No newline at end of file diff --git a/services/rest_gate/.hosts b/services/rest_gate/.hosts deleted file mode 100644 index ee7578e..0000000 --- a/services/rest_gate/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.83 rest.LOCAL_DOMAIN diff --git a/services/rest_gate/.int_test.env b/services/rest_gate/.int_test.env deleted file mode 120000 index 582b6a2..0000000 --- a/services/rest_gate/.int_test.env +++ /dev/null @@ -1 +0,0 @@ -../../.int_test.env \ No newline at end of file diff --git a/services/rest_gate/cfg/config.yml b/services/rest_gate/cfg/config.yml deleted file mode 100644 index 0acdad3..0000000 --- a/services/rest_gate/cfg/config.yml +++ /dev/null @@ -1,12 +0,0 @@ -prometheus: - enabled: true - address: :9090 - -server: - # The IP and port to listen on. - listen-address: 0.0.0.0:8090 - -# Wallet settings -wallet: - path: /wallet.json # Path to wallet - passphrase: one # Password to decrypt wallet diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml deleted file mode 100644 index 09cfe38..0000000 --- a/services/rest_gate/docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- - -services: - rest_gate: - image: ${REST_GW_IMAGE}:${REST_GW_VERSION} - domainname: ${LOCAL_DOMAIN} - hostname: rest - container_name: rest_gate - restart: on-failure - networks: - rest_gate_int: - internet: - ipv4_address: ${IPV4_PREFIX}.83 - volumes: - - ./wallet.json:/wallet.json - - ./../../vendor/hosts:/etc/hosts - - ./cfg:/etc/frostfs/rest - stop_signal: SIGTERM - stop_grace_period: 15s - env_file: [ ".env", ".int_test.env" ] - command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ] - environment: - - REST_GW_POOL_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 - -networks: - rest_gate_int: - internet: - external: true - name: basenet_internet diff --git a/services/rest_gate/wallet.json b/services/rest_gate/wallet.json deleted file mode 100644 index 2b60501..0000000 --- a/services/rest_gate/wallet.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "version": "3.0", - "accounts": [ - { - "address": "NPFCqWHfi9ixCJRu7DABRbVfXRbkSEr9Vo", - "key": "6PYTAGjdaeicUDPqGv9mmgwb9kTwimWJJmmfNqJSDGH9qM79zSRcL9oHiB", - "label": "REST Gateway", - "contract": { - "script": "DCECcuPzZCZ2VyDsm2jKEOMnU6xEWO2bF1dvOvBWTDFYB1ZBVuezJw==", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "isDefault": false - } - ], - "scrypt": { - "n": 16384, - "r": 8, - "p": 8 - }, - "extra": { - "Tokens": null - } -} From c4d4fecb89ee1fbd2f7f02254f2ebf8264b47553 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 27 Dec 2024 10:42:19 +0300 Subject: [PATCH 72/81] [#96] s3_lifecycler: Fix docker-compose warnings Signed-off-by: Evgenii Stratonikov --- services/s3_lifecycler/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/services/s3_lifecycler/docker-compose.yml b/services/s3_lifecycler/docker-compose.yml index 3456d0c..2fffa50 100644 --- a/services/s3_lifecycler/docker-compose.yml +++ b/services/s3_lifecycler/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: s3_lifecycler: image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION} From 90147c71081361e4348f0a041848e7ccf08d1235 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 27 Dec 2024 09:33:04 +0300 Subject: [PATCH 73/81] [#95] Output errors during `make clean` It happens that a volume may not be deleted during `make clean`. For example, if the volume is being used by a container. If this happens, there are no errors printed to stdout. And old volumes may cause errors during subsequent `make up` Signed-off-by: Nikita Zinkevich --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 58a12e0..f24ec9d 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ clean: vols=`docker-compose -f services/$${svc}/docker-compose.yml config --volumes` if [[ ! -z "$${vols}" ]]; then for vol in $${vols}; do - docker volume rm -f "$${svc}_$${vol}" 2> /dev/null + docker volume rm -f "$${svc}_$${vol}" done fi done From 5471dbfc0e57babdc5f95c1c788fbd3a61c9e9d7 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Thu, 9 Jan 2025 11:23:42 +0300 Subject: [PATCH 74/81] [#98] s3_gate: Fix custom user wallets folder creation during compose up Make custom wallets volume to point to the `wallets` directory in the project's root. Signed-off-by: Nikita Zinkevich --- README.md | 9 +++++---- services/s3_gate/docker-compose.yml | 6 ++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8e4b714..415885e 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Registers user wallet and issues s3 credentials. Usage and default parameter values: ```sh -make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] +make s3cred [password=""] [contract_password=s3] [wallet=""] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] ``` As soon as the storage node is in the network map (see above) you can generate S3 @@ -159,8 +159,9 @@ $ make s3cred "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT" } ``` -Running without any parameters will result in defaults which are based on the private key from -`/user-wallet.json` file and `/wallet.json` contract wallet. +Running without any parameters results in defaults which are based on the private key from +`/wallets/wallet.json` user wallet and `/wallet.json` contract wallet. +If `wallet` parameter is set, gate searches custom user wallet file in `/wallets` directory. Now let's configure an S3 client (AWS CLI will be used as example): @@ -172,7 +173,7 @@ Default region name []: us-east-1 Default output format []: json ``` -If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter. +If you need to create credentials for different users, put user wallet to `wallets` dir and specify it via `wallet` parameter. Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params. ```sh diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index f7fc28d..3c1f9f6 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -14,10 +14,8 @@ services: volumes: # Gate wallet - ./wallet.json:/wallet.json - # Custom user wallets - - ./wallets:/wallets - # Default user wallet - - ./../../wallets/wallet.json:/wallets/wallet.json + # Folder for custom user wallets + - ./../../wallets/:/wallets/ - ./tls.key:/tls.key - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts From 0981202d6bcc0c454d55cfb1cb5c9b355e217170 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:47:17 +0300 Subject: [PATCH 75/81] [#100] Use docker images as vendor binary source Signed-off-by: Alex Vanin --- .env | 6 ++++-- services/ir/artifacts.mk | 8 +++----- services/morph_chain/artifacts.mk | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 824a4db..053f179 100644 --- a/.env +++ b/.env @@ -36,11 +36,13 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli +FROSTFS_CLI_VERSION=0.44.4 +FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm +FROSTFS_ADM_VERSION=0.44.4 +FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index 0cdbdbb..1c2cf5d 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -30,11 +30,9 @@ get.cli: @mkdir -p ./vendor ifeq (${FROSTFS_CLI_PATH},) - @echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}" - @curl \ - -ksSL "${FROSTFS_CLI_URL}" \ - -o ${FROSTFS_CLI_FILE} - @chmod +x ${FROSTFS_CLI_FILE} + echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}" + $(shell docker cp `docker create --name tmp ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}`:/bin/frostfs-cli ${FROSTFS_CLI_FILE} && docker rm tmp >/dev/null) + chmod +x ${FROSTFS_CLI_FILE} else @echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}" @cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE} diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 77a7ae3..4a4504a 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -23,8 +23,8 @@ get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm get.adm: ifeq (${FROSTFS_ADM_PATH},) - @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}" - @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST} + @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}" + $(shell docker cp `docker create --name tmp ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}`:/bin/frostfs-adm ${FROSTFS_ADM_DEST} && docker rm tmp >/dev/null) @chmod +x ${FROSTFS_ADM_DEST} else @echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}" From ea91d8823274fa5eac9bd86a49369c6267a3fc6f Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:54:24 +0300 Subject: [PATCH 76/81] [#100] Add target to prepare storage for service components Signed-off-by: Alex Vanin --- Makefile | 22 ++++++++++++++++++++-- cli-cfg.yml | 3 +++ services/morph_chain/docker-compose.yml | 1 + wallets/system-wallet.json | 1 + 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 cli-cfg.yml create mode 100644 wallets/system-wallet.json diff --git a/Makefile b/Makefile index f24ec9d..b3b342d 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,8 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) # Start environment .PHONY: up -up: up/basic +up: up/basic up/pre-services @$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet" @echo "Full FrostFS Developer Environment is ready" # Build up FrostFS @@ -98,6 +97,25 @@ up/bootstrap: get vendor/hosts || die "Failed to create subject for the wallet"; \ fi echo "FrostFS sidechain environment is deployed" + +# Prepare to start services +.PHONY: up/pre-services +up/pre-services: + @source ./bin/helper.sh + @echo "Prepare storage for services"; \ + if [ -z "$$(./vendor/frostfs-cli -c cli-cfg.yml container list)" ]; then \ + subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | tail -1 | tr -d ' \r\n'` \ + && echo "Subject key: $${subj_key}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name system \ + || die "Failed to create subject for system wallet"; \ + proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | head -1 | cut -d" " -f1` \ + && echo "Proxy acc: $${proxy_acc}" \ + && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "CORS Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container" + fi + @echo "Storage is prepared"; # Build up certain service .PHONY: up/% diff --git a/cli-cfg.yml b/cli-cfg.yml new file mode 100644 index 0000000..9a5c7eb --- /dev/null +++ b/cli-cfg.yml @@ -0,0 +1,3 @@ +wallet: ./wallets/system-wallet.json +password: "" +rpc-endpoint: s01.frostfs.devenv:8080 diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 6b83716..c3a1481 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,6 +19,7 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - ./../../wallets/system-wallet.json:/wallets/system-wallet.json - ./../s3_gate/wallet.json:/wallets/s3-wallet.json - ./../storage/wallet01.json:/wallets/storage/wallet01.json - ./../storage/wallet02.json:/wallets/storage/wallet02.json diff --git a/wallets/system-wallet.json b/wallets/system-wallet.json new file mode 100644 index 0000000..2f57db1 --- /dev/null +++ b/wallets/system-wallet.json @@ -0,0 +1 @@ +{"version":"1.0","accounts":[{"address":"NQijiVKHbL22PfF2AJQukv1CX75itxgzht","key":"6PYQKrpme57VqaucxuF7dDoSZRRA8d94oatHcScqhiFBauCXQvFDaYwEWa","label":"","contract":{"script":"DCEDRdLtpFIWeYyI7doTKRhIl4qYjaybGDveTyGpbqjsLZNBVuezJw==","parameters":[{"name":"parameter0","type":"Signature"}],"deployed":false},"lock":false,"isDefault":false}],"scrypt":{"n":16384,"r":8,"p":8},"extra":{"Tokens":null}} \ No newline at end of file From e0141c11545ecf107bf50fe5da293a1390ba2695 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:55:43 +0300 Subject: [PATCH 77/81] [#100] Update components to the latest versions Signed-off-by: Alex Vanin --- .env | 16 ++++++++-------- services/http_gate/cfg/config.yml | 3 +++ services/s3_gate/cfg/config.yml | 3 +++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.env b/.env index 053f179..dec29ce 100644 --- a/.env +++ b/.env @@ -12,20 +12,20 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.44.4 +IR_VERSION=0.45.0-rc.3 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.44.4 +NODE_VERSION=0.45.0-rc.3 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.32.0 -HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw +HTTP_GW_VERSION=0.33.0-rc.2 +HTTP_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-http-gw # S3 Gate -S3_GW_VERSION=0.32.0 -S3_GW_IMAGE=truecloudlab/frostfs-s3-gw +S3_GW_VERSION=0.33.0-rc.2 +S3_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-s3-gw # Lifecycler S3_LIFECYCLER_VERSION=0.1.3 @@ -36,12 +36,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_VERSION=0.44.4 +FROSTFS_CLI_VERSION=0.45.0-rc.3 FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=0.44.4 +FROSTFS_ADM_VERSION=0.45.0-rc.3 FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary diff --git a/services/http_gate/cfg/config.yml b/services/http_gate/cfg/config.yml index 1684c58..2f15e44 100644 --- a/services/http_gate/cfg/config.yml +++ b/services/http_gate/cfg/config.yml @@ -22,3 +22,6 @@ server: wallet: path: /wallet.json # Path to wallet passphrase: one # Passphrase to decrypt wallet + +containers: + cors: cors.container diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 03e84e0..3d8e400 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -47,3 +47,6 @@ frostfsid: policy: enabled: false + +containers: + cors: cors.container From 018e33c1f9048df8e6c580bde535df535a297838 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 27 Mar 2025 17:34:37 +0300 Subject: [PATCH 78/81] [#101] Fix s3 gateway initialization Was broken in #100 Signed-off-by: Alex Vanin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b3b342d..fcc09a3 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ up/pre-services: && echo "Subject key: $${subj_key}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name system \ || die "Failed to create subject for system wallet"; \ - proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | head -1 | cut -d" " -f1` \ + proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | cut -d" " -f1` \ && echo "Proxy acc: $${proxy_acc}" \ && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ From 6ef4d4aa0ad7f99f948c0cf19bda4f0faa191c37 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Mar 2025 14:07:50 +0300 Subject: [PATCH 79/81] [#102] Makefile: Parallelize GAS transfer The last version of the `frostfs-adm` allows to work with multiple wallets in `refill-gas` and `proxy-add-account` commands. This makes `up/bootstrap` target take ~6s less time. The Makefile stuff is not obvious, but it seem to work as expected. Refs TrueCloudLab/frostfs-node#1590, TrueCloudLab/frostfs-node#1578 Signed-off-by: Evgenii Stratonikov --- Makefile | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index fcc09a3..911d168 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,8 @@ up/basic: up/bootstrap # Start bootstrap services .PHONY: up/bootstrap +up/bootstrap: STORAGE_WALLETS = $(wildcard ./services/storage/wallet*.json) +up/bootstrap: STORAGE_ACCOUNTS = $(foreach wallet,$(STORAGE_WALLETS),$(shell docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$(notdir $(wallet)) | head -1 | awk '{print $$1}' )) up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh @@ -80,13 +82,16 @@ up/bootstrap: get vendor/hosts @./vendor/frostfs-adm --config frostfs-adm.yml morph \ ape add-rule-chain --target-type namespace --target-name "" \ --rule 'allow Container.* *' --chain-id "allow_container_ops" - @for f in ./services/storage/wallet*.json; do \ - echo "Transfer GAS to wallet $${f}" \ - && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ - || die "Failed to transfer GAS to alphabet wallets"; \ - echo "Register storage wallet $${f} in proxy contract" \ - && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$${f##*/} | head -1 | awk '{print $1}'` || die "Couldn't set storage allet as proxy wallet" - done + + echo -e "Transfer GAS to storage wallets: $(foreach wallet,$(STORAGE_WALLETS),\n\t$(wallet))" + ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas \ + $(foreach wallet,$(STORAGE_WALLETS),--storage-wallet $(wallet)) \ + --gas 10.0 \ + || die "Failed to transfer GAS to alphabet wallets" + echo -e "Register storage accounts in proxy contract: $(foreach account,$(STORAGE_ACCOUNTS),\n\t$(account))" + ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml \ + $(foreach account,$(STORAGE_ACCOUNTS),--account=$(account)) \ + || die "Couldn't set storage allet as proxy wallet" @echo "Create frostfsid subject for ./wallets/wallet.json"; \ if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ echo "Subject already exists"; \ From 8608ba46ff2b6bc07a3d662ce0ace00bd137a31e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 8 Apr 2025 14:51:37 +0300 Subject: [PATCH 80/81] [#104] Update components to the latest versions New S3 gateway version supports MFADelete feature that requires container with MFA objects. Signed-off-by: Alex Vanin --- .env | 12 ++++++------ Makefile | 5 ++++- services/s3_gate/cfg/config.yml | 1 + 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.env b/.env index dec29ce..9c4531d 100644 --- a/.env +++ b/.env @@ -12,19 +12,19 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.45.0-rc.3 +IR_VERSION=0.45.0-rc.6 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.45.0-rc.3 +NODE_VERSION=0.45.0-rc.6 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.33.0-rc.2 +HTTP_GW_VERSION=0.33.0-rc.3 HTTP_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-http-gw # S3 Gate -S3_GW_VERSION=0.33.0-rc.2 +S3_GW_VERSION=0.33.0-rc.3 S3_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-s3-gw # Lifecycler @@ -36,12 +36,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_VERSION=0.45.0-rc.3 +FROSTFS_CLI_VERSION=0.45.0-rc.6 FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=0.45.0-rc.3 +FROSTFS_ADM_VERSION=0.45.0-rc.6 FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary diff --git a/Makefile b/Makefile index 911d168..56e5f7c 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,10 @@ up/pre-services: && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ && echo "CORS Container: $${cid}" \ - && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container" + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container"; \ + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "mfa" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "MFA Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create MFA container" fi @echo "Storage is prepared"; diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 3d8e400..369325c 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -50,3 +50,4 @@ policy: containers: cors: cors.container + mfa: mfa.container From df8249f2aeabbbe8caab4229e42321027269cb74 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 11 Apr 2025 15:49:20 +0300 Subject: [PATCH 81/81] [#105] Add website container creation Signed-off-by: Nikita Zinkevich --- Makefile | 3 +++ services/s3_gate/cfg/config.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 56e5f7c..1f97c5f 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,9 @@ up/pre-services: cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "mfa" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ && echo "MFA Container: $${cid}" \ && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create MFA container" + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "website" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "Website configuration Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create Website configuration container" fi @echo "Storage is prepared"; diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 369325c..e5c0b53 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -51,3 +51,4 @@ policy: containers: cors: cors.container mfa: mfa.container + website: website.container