From 40454b55076c9590d9e9ded5a85069820cf88cbd Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 15 Nov 2023 16:08:15 +0300 Subject: [PATCH] [#56] Add loki Signed-off-by: Alexander Chuprov --- .env | 8 +- services/grafana/.hosts | 1 + services/grafana/docker-compose.yml | 8 + .../dashboards/storage-node-logs.json | 358 ++++++++++++++++++ .../provisioning/datasources/datasource.yml | 5 + services/storage/cfg/config.yml | 7 +- 6 files changed, 384 insertions(+), 3 deletions(-) create mode 100644 services/grafana/provisioning/dashboards/storage-node-logs.json diff --git a/.env b/.env index de7b691..0969ff5 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.103.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=365a7ca0 +IR_VERSION=0.37.0-rc.1-93-g364f835b IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=365a7ca0 +NODE_VERSION=0.37.0-rc.1-93-g364f835b NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -64,3 +64,7 @@ PROMETHEUS_IMAGE=prom/prometheus # Grafana versions GRAFANA_VERSION=9.5.6 GRAFANA_IMAGE=grafana/grafana + +# Loki versions +LOKI_VERSION=2.9.1 +LOKI_IMAGE=grafana/loki \ No newline at end of file diff --git a/services/grafana/.hosts b/services/grafana/.hosts index 8f36ea6..537a644 100644 --- a/services/grafana/.hosts +++ b/services/grafana/.hosts @@ -1 +1,2 @@ IPV4_PREFIX.122 grafana.LOCAL_DOMAIN +IPV4_PREFIX.123 loki.LOCAL_DOMAIN diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 6d27299..90b67f2 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -17,6 +17,14 @@ services: stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] + loki: + image: ${LOKI_IMAGE}:${LOKI_VERSION} + command: -config.file=/etc/loki/local-config.yaml + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.123 + networks: grafana_int: internet: diff --git a/services/grafana/provisioning/dashboards/storage-node-logs.json b/services/grafana/provisioning/dashboards/storage-node-logs.json new file mode 100644 index 0000000..3c91d52 --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node-logs.json @@ -0,0 +1,358 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({hostname=~\"$hostname\"} |~ \"$level\" |~ `$filter` [1s])", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log intensity", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 226 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.width", + "value": 424 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\"} |~ `$filter` | json level |~ \"$level\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "id": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Hostname", + "multi": true, + "name": "hostname", + "options": [], + "query": { + "label": "hostname", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "debug", + "value": "debug" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "warn", + "value": "warn" + }, + { + "selected": false, + "text": "error", + "value": "error" + }, + { + "selected": false, + "text": "dpanic", + "value": "dpanic" + }, + { + "selected": false, + "text": "panic", + "value": "panic" + }, + { + "selected": false, + "text": "fatal", + "value": "fatal" + } + ], + "query": "debug,info,warn,error,dpanic,panic,fatal", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Filter", + "name": "filter", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node logs", + "uid": "bef4b42d-c74e-4241-96cb-d239113f44bc", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml index 81b6a7b..7d7427f 100644 --- a/services/grafana/provisioning/datasources/datasource.yml +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -6,3 +6,8 @@ datasources: access: proxy orgId: 1 url: http://prometheus:9090 +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 \ No newline at end of file diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 0a85096..6b3e7a8 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -1,6 +1,11 @@ # Logger section logger: level: debug # Minimum enabled logging level + loki: + enabled: true + endpoint: "loki.frostfs.devenv:3100/api/prom/push" + max_batch_delay: 1s + max_batch_size: 200 # Profiler section pprof: @@ -14,7 +19,7 @@ prometheus: address: :9090 # Server address shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown - # Application tracing section +# Application tracing section tracing: enabled: true exporter: otlp_grpc