diff --git a/.bootstrap_services b/.bootstrap_services index a56729d..59b770a 100644 --- a/.bootstrap_services +++ b/.bootstrap_services @@ -4,4 +4,5 @@ basenet morph_chain jaeger prometheus +pyroscope grafana diff --git a/.env b/.env index e04d5ca..95099f4 100644 --- a/.env +++ b/.env @@ -64,3 +64,7 @@ PROMETHEUS_IMAGE=prom/prometheus # Grafana versions GRAFANA_VERSION=9.5.6 GRAFANA_IMAGE=grafana/grafana + +# Pyroscope version +PYROSCOPE_VERSION=1.0.0-rc.0 +PYROSCOPE_IMAGE=grafana/pyroscope diff --git a/services/grafana/provisioning/dashboards/pprof.json b/services/grafana/provisioning/dashboards/pprof.json new file mode 100644 index 0000000..8c55081 --- /dev/null +++ b/services/grafana/provisioning/dashboards/pprof.json @@ -0,0 +1,148 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 10, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "phlare", + "uid": "P02E4190217B50628" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "targets": [ + { + "datasource": { + "type": "phlare", + "uid": "P02E4190217B50628" + }, + "groupBy": [], + "labelSelector": "{instance=\"${_instance}\"}", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "queryType": "profile", + "refId": "A" + } + ], + "title": "Inuse objects", + "type": "flamegraph" + }, + { + "datasource": { + "type": "phlare", + "uid": "P02E4190217B50628" + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "targets": [ + { + "datasource": { + "type": "phlare", + "uid": "P02E4190217B50628" + }, + "groupBy": [], + "labelSelector": "{instance=\"${_instance}\"}", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "queryType": "profile", + "refId": "A" + } + ], + "title": "CPU", + "type": "flamegraph" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "morph_chain:20011", + "value": "morph_chain:20011" + }, + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "_instance", + "options": [ + { + "selected": false, + "text": "s01:6060", + "value": "s01:6060" + }, + { + "selected": false, + "text": "s02:6060", + "value": "s02:6060" + }, + { + "selected": false, + "text": "s03:6060", + "value": "s03:6060" + }, + { + "selected": false, + "text": "s04:6060", + "value": "s04:6060" + }, + { + "selected": false, + "text": "s3:6060", + "value": "s3:6060" + }, + { + "selected": true, + "text": "morph_chain:20011", + "value": "morph_chain:20011" + } + ], + "query": "s01:6060,s02:6060,s03:6060,s04:6060,s3:6060,morph_chain:20011", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Pprof", + "uid": "eba3003c-e846-400c-b7d3-6658eddc28bc", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml index 81b6a7b..24f9ce2 100644 --- a/services/grafana/provisioning/datasources/datasource.yml +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -6,3 +6,9 @@ datasources: access: proxy orgId: 1 url: http://prometheus:9090 +- name: Pyroscope + type: phlare + url: http://pyroscope:4100 + jsonData: + minStep: '15s' + backendType: 'pyroscope' \ No newline at end of file diff --git a/services/pyroscope/.env b/services/pyroscope/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/pyroscope/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/pyroscope/.hosts b/services/pyroscope/.hosts new file mode 100644 index 0000000..96c8a92 --- /dev/null +++ b/services/pyroscope/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.123 pyroscope.LOCAL_DOMAIN diff --git a/services/pyroscope/.int_test.env b/services/pyroscope/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/pyroscope/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/pyroscope/docker-compose.yml b/services/pyroscope/docker-compose.yml new file mode 100644 index 0000000..de0938f --- /dev/null +++ b/services/pyroscope/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3.9' +services: + pyroscope: + image: grafana/pyroscope:latest + domainname: ${LOCAL_DOMAIN} + hostname: pyroscope + container_name: pyroscope + networks: + pyroscope_int: + internet: + ipv4_address: ${IPV4_PREFIX}.123 + ports: + - '4100:4100' + command: + - -config.file=/etc/pyroscope/config.yaml + - server + volumes: + - ./server.yml:/etc/pyroscope/config.yaml + +networks: + pyroscope_int: + internet: + external: true + name: basenet_internet diff --git a/services/pyroscope/server.yml b/services/pyroscope/server.yml new file mode 100644 index 0000000..1cabd5a --- /dev/null +++ b/services/pyroscope/server.yml @@ -0,0 +1,32 @@ +--- +server: + http_listen_port: 4100 + +scrape_configs: + - job_name: "default" + scrape_interval: "15s" + static_configs: + - targets: ["127.0.0.1:4100"] + labels: + service_name: "pyroscope" + - job_name: storage + scrape_interval: "15s" + static_configs: + - targets: ["s01:6060", "s02:6060", "s03:6060"] + labels: + service_name: "storage" + - targets: ["s04:6060"] + labels: + service_name: "s04" + - job_name: s3 + scrape_interval: "15s" + static_configs: + - targets: ["s3:6060"] + labels: + service_name: "s3" + - job_name: morph + scrape_interval: "15s" + static_configs: + - targets: ["morph_chain:20011"] + labels: + service_name: "morph" \ No newline at end of file diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 7fa041a..3091520 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -5,6 +5,10 @@ prometheus: enabled: true address: :9090 +pprof: + enabled: true + address: :6060 + # Interval to check node health rebalance_interval: 30s