diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 0000000..9ab627a --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:1.21 as builder + +ARG BUILD=now +ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler +ARG VERSION=dev + +WORKDIR /src +COPY . /src + +RUN make + +# Executable image +FROM alpine AS frostfs-s3-lifecycler +RUN apk add --no-cache bash ca-certificates + +WORKDIR / + +COPY --from=builder /src/bin/frostfs-s3-lifecycler /bin/frostfs-s3-lifecycler + +ENTRYPOINT ["/bin/frostfs-s3-lifecycler"] diff --git a/.docker/Dockerfile.dirty b/.docker/Dockerfile.dirty new file mode 100644 index 0000000..86eb5c9 --- /dev/null +++ b/.docker/Dockerfile.dirty @@ -0,0 +1,8 @@ +FROM alpine AS frostfs-s3-lifecycler +RUN apk add --no-cache bash ca-certificates + +WORKDIR / + +COPY /bin/frostfs-s3-lifecycler /bin/frostfs-s3-lifecycler + +ENTRYPOINT ["/bin/frostfs-s3-lifecycler"] diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..09348a5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +.cache +.github diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..446331f --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.idea +.cache +bin +temp +/plugins/ +/vendor/ +metrics-dump.json diff --git a/.gitlint b/.gitlint new file mode 100644 index 0000000..e7218ac --- /dev/null +++ b/.gitlint @@ -0,0 +1,11 @@ +[general] +fail-without-commits=True +regex-style-search=True +contrib=CC1 + +[title-match-regex] +regex=^\[\#[0-9Xx]+\]\s + +[ignore-by-title] +regex=^Release(.*) +ignore=title-match-regex diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..43a66a6 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,67 @@ +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 15m + + # include test files or not, default is true + tests: true + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: tab + +# all available settings of specific linters +linters-settings: + exhaustive: + # indicates that switch statements are to be considered exhaustive if a + # 'default' case is present, even if all enum members aren't listed in the + # switch + default-signifies-exhaustive: true + govet: + # report about shadowed variables + check-shadowing: false + custom: + truecloudlab-linters: + path: bin/external_linters.so + original-url: git.frostfs.info/TrueCloudLab/linters.git + settings: + noliteral: + enable: true + target-methods: ["Fatal"] + disable-packages: [] + constants-package: "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" + +linters: + enable: + # mandatory linters + - govet + - revive + + # some default golangci-lint linters + - errcheck + - gosimple + - ineffassign + - staticcheck + - typecheck + - unused + + # extra linters + - exhaustive + - godot + - gofmt + - whitespace + - goimports + - truecloudlab-linters + disable-all: true + fast: false + +issues: + include: + - EXC0002 # should have a comment + - EXC0003 # test/Test ... consider calling this + - EXC0004 # govet + - EXC0005 # C-style breaks diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3c963be --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,52 @@ +ci: + autofix_prs: false + +repos: + - repo: https://github.com/jorisroovers/gitlint + rev: v0.19.1 + hooks: + - id: gitlint + stages: [commit-msg] + - id: gitlint-ci + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-merge-conflict + - id: check-json + - id: check-xml + - id: check-yaml + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + exclude: ".key$" + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.2 + hooks: + - id: shellcheck + + - repo: local + hooks: + - id: make-lint-install + name: install linters + entry: make lint-install + language: system + pass_filenames: false + + - id: make-lint + name: run linters + entry: make lint + language: system + pass_filenames: false + + - id: go-unit-tests + name: go unit tests + entry: make test + pass_filenames: false + types: [go] + language: system diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c8c80eb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +This document outlines major changes between releases. + +## [Unreleased] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..09d65e1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,156 @@ +# Contribution guide + +First, thank you for contributing! We love and encourage pull requests from +everyone. Please follow the guidelines: + +- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/issues) and + [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/pulls) for existing + discussions. + +- Open an issue first, to discuss a new feature or enhancement. + +- Write tests and make sure the test suite passes locally and on CI. + +- Open a pull request and reference the relevant issue(s). + +- Make sure your commits are logically separated and have good comments + explaining the details of your change. + +- After receiving a feedback, amend your commits or add new ones as + appropriate. + +- **Have fun!** + +## Development Workflow + +Start by forking the `frostfs-s3-lifecycler` repository, make changes in a branch and then +send a pull request. We encourage pull requests to discuss code changes. Here +are the steps in details: + +### Set up your git repository +Fork [FrostFS S3 Gateway +upstream](https://git.frostfs.info/repo/fork/15) source repository +to your own personal repository. Copy the URL of your fork (you will need it for +the `git clone` command below). + +```sh +$ git clone https://git.frostfs.info//frostfs-s3-lifecycler.git +``` + +### Set up git remote as ``upstream`` +```sh +$ cd frostfs-s3-lifecycler +$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler.git +$ git fetch upstream +$ git merge upstream/master +... +``` + +### Create your feature branch +Before making code changes, make sure you create a separate branch for these +changes. Maybe you will find it convenient to name a branch in +`/-` format. + +``` +$ git checkout -b feature/123-something_awesome +``` + +### Test your changes +After your code changes, make sure + +- To add test cases for the new code. +- To run `make lint` +- To squash your commits into a single commit or a series of logically separated + commits with `git rebase -i`. It's okay to force update your pull request. +- To run `make test` and `make all` successfully. + +### Commit changes +After verification, commit your changes. There is a [great +post](https://chris.beams.io/posts/git-commit/) on how to write useful commit +messages. Try following this template: + +``` +[#Issue] Summary + +Description + + + + +``` + +``` +$ git commit -ams '[#123] Add some feature' +``` + +### Push to the branch +Push your locally committed changes to the remote origin (your fork) +``` +$ git push origin feature/123-something_awesome +``` + +### Create a Pull Request +Pull requests can be created via Forgejo. Refer to [this +document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for +detailed steps on how to create a pull request. After a Pull Request gets peer +reviewed and approved, it will be merged. + +## DCO Sign off + +All authors to the project retain copyright to their work. However, to ensure +that they are only submitting work that they have rights to, we require +everyone to acknowledge this by signing their work. + +Any copyright notices in this repository should specify the authors as "the +contributors". + +To sign your work, just add a line like this at the end of your commit message: + +``` +Signed-off-by: Samii Sakisaka +``` + +This can be easily done with the `--signoff` option to `git commit`. + +By doing this you state that you can certify the following (from [The Developer +Certificate of Origin](https://developercertificate.org/)): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` diff --git a/CREDITS.md b/CREDITS.md new file mode 100644 index 0000000..5ba4f37 --- /dev/null +++ b/CREDITS.md @@ -0,0 +1,11 @@ +# Credits + +In alphabetical order: + +- Denis Kirillov (@dkirillov) + +# Contributors + +In chronological order: + +- Denis Kirillov (@dkirillov) diff --git a/Makefile b/Makefile new file mode 100755 index 0000000..1e8a254 --- /dev/null +++ b/Makefile @@ -0,0 +1,166 @@ +#!/usr/bin/make -f + +REPO ?= $(shell go list -m) +VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") +GO_VERSION ?= 1.21 +LINT_VERSION ?= 1.56.1 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.5 +BINDIR = bin + +METRICS_DUMP_OUT ?= ./metrics-dump.json + +CMDS = $(notdir $(basename $(wildcard cmd/*))) +BINS = $(addprefix $(BINDIR)/, $(CMDS)) + +# Variables for docker +REPO_BASENAME = $(shell basename `go list -m`) +HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)" +HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" + +OUTPUT_LINT_DIR ?= $(shell pwd)/bin +LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION) +TMP_DIR := .cache + +# Make all binaries +.PHONY: all +all: $(BINS) + +.PHONY: $(BINS) +$(BINS): $(BINDIR) dep fmts + @echo "⇒ Build $@" + CGO_ENABLED=0 \ + go build -v -trimpath \ + -ldflags "-X main.Version=$(VERSION)" \ + -o bin/frostfs-s3-lifecycler ./cmd/$(notdir $@) + +.PHONY: $(BINDIR) +$(BINDIR): + @echo "⇒ Ensure dir: $@" + @mkdir -p $@ + +# Pull go dependencies +.PHONY: dep +dep: + @printf "⇒ Download requirements: " + @CGO_ENABLED=0 \ + go mod download && echo OK + @printf "⇒ Tidy requirements: " + @CGO_ENABLED=0 \ + go mod tidy -v && echo OK + +.PHONY: image +image: + @echo "⇒ Build FrostFS S3 Lifecycler docker image " + @docker build \ + --build-arg REPO=$(REPO) \ + --build-arg VERSION=$(VERSION) \ + --rm \ + -f .docker/Dockerfile \ + -t $(HUB_IMAGE):$(HUB_TAG) . + +.PHONY: image-push +image-push: + @echo "⇒ Publish image" + @docker push $(HUB_IMAGE):$(HUB_TAG) + +.PHONY: dirty-image +dirty-image: + @echo "⇒ Build FrostFS S3 Lifecycler dirty docker image " + @docker build \ + --build-arg REPO=$(REPO) \ + --build-arg VERSION=$(VERSION) \ + --rm \ + -f .docker/Dockerfile.dirty \ + -t $(HUB_IMAGE)-dirty:$(HUB_TAG) . + +.PHONY: docker/ +docker/%: + $(if $(filter $*,all $(BINS)), \ + @echo "=> Running 'make $*' in clean Docker environment" && \ + docker run --rm -t \ + -v `pwd`:/src \ + -w /src \ + -u `stat -c "%u:%g" .` \ + --env HOME=/src \ + golang:$(GO_VERSION) make $*,\ + @echo "supported docker targets: all $(BINS) lint") + +# Run tests +.PHONY: test +test: + @go test ./... -cover + +# Run tests with race detection and produce coverage output +.PHONY: cover +cover: + @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic + @go tool cover -html=coverage.txt -o coverage.html + +# Run all code formatters +.PHONY: fmts +fmts: fmt imports + +# Reformat code +.PHONY: fmt +fmt: + @echo "⇒ Processing gofmt check" + @GO111MODULE=on gofmt -s -w ./ + +# Reformat imports +.PHONY: imports +imports: + @echo "⇒ Processing goimports check" + @GO111MODULE=on goimports -w ./ + +# Install linters +.PHONY: lint-install +lint-install: + @if [ ! -d "$(LINT_DIR)" ]; then \ + mkdir -p $(TMP_DIR); \ + rm -rf $(TMP_DIR)/linters; \ + git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters; \ + make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR); \ + rm -rf $(TMP_DIR)/linters; \ + rmdir $(TMP_DIR) 2>/dev/null || true; \ + CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION); \ + fi + +# Run linters +.PHONY: lint +lint: lint-install + $(LINT_DIR)/golangci-lint --timeout=5m run + +# Run linters in Docker +.PHONY: docker/lint +docker/lint: + docker run --rm -it \ + -v `pwd`:/src \ + -u `stat -c "%u:%g" .` \ + --env HOME=/src \ + golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint' + +# Activate pre-commit hooks +.PHONY: pre-commit +pre-commit: + pre-commit install -t pre-commit -t commit-msg + +# Deactivate pre-commit hooks +.PHONY: unpre-commit +unpre-commit: + pre-commit uninstall -t pre-commit -t commit-msg + +.PHONY: clean +clean: + @rm -rf $(DIRS) + +# Show current version +.PHONY: version +version: + @echo $(VERSION) + +# Dump metrics (use METRICS_DUMP_OUT variable to override default out file './metrics-dump.json') +.PHONY: dump-metrics +dump-metrics: + @go test ./internal/metrics -run TestDescribeAll --tags=dump_metrics --out=$(abspath $(METRICS_DUMP_OUT)) + +include help.mk diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..b82608c --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v0.1.0 diff --git a/cmd/s3-lifecycler/app.go b/cmd/s3-lifecycler/app.go new file mode 100644 index 0000000..bbe27b2 --- /dev/null +++ b/cmd/s3-lifecycler/app.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/metrics" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +type ( + App struct { + log *zap.Logger + logLevel zap.AtomicLevel + cfg *viper.Viper + done chan struct{} + appServices []*metrics.Service + appMetrics *metrics.AppMetrics + } +) + +const ( + HealthStatusUndefined int32 = 0 + HealthStatusStarting int32 = 1 + HealthStatusReady int32 = 2 + HealthStatusShuttingDown int32 = 3 +) + +func newApp(cfg *viper.Viper, log *zap.Logger, level zap.AtomicLevel) *App { + a := &App{ + log: log, + logLevel: level, + cfg: cfg, + done: make(chan struct{}), + appMetrics: metrics.NewAppMetrics(), + } + a.appMetrics.SetHealth(HealthStatusStarting) + + return a +} + +func (a *App) Wait() { + a.log.Info(logs.ApplicationStarted, + zap.String("app_name", "frostfs-s3-lifecycler"), + zap.String("version", Version)) + + a.appMetrics.SetHealth(HealthStatusReady) + a.appMetrics.SetVersion(Version) + + <-a.done + + a.log.Info(logs.ApplicationStopped) +} + +func (a *App) Serve(ctx context.Context) { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGHUP) + + a.startAppServices() + +loop: + for { + select { + case <-ctx.Done(): + break loop + case <-sigs: + a.configReload() + } + } + + a.log.Info(logs.StoppingApplication) + + a.appMetrics.SetHealth(HealthStatusShuttingDown) + a.stopAppServices() + + close(a.done) +} + +func (a *App) configReload() { + a.log.Info(logs.SIGHUPConfigReloadStarted) + if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) { + a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) + return + } + if err := readInConfig(a.cfg); err != nil { + a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) + return + } + + if lvl, err := getLogLevel(a.cfg.GetString(cfgLoggerLevel)); err != nil { + a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) + } else { + a.logLevel.SetLevel(lvl) + } + + a.stopAppServices() + a.startAppServices() + + a.log.Info(logs.SIGHUPConfigReloadCompleted) +} + +func (a *App) startAppServices() { + a.appServices = a.appServices[:0] + + pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} + pprofService := metrics.NewPprofService(a.log, pprofConfig) + a.appServices = append(a.appServices, pprofService) + go pprofService.Start() + + prometheusConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPrometheusEnabled), Address: a.cfg.GetString(cfgPrometheusAddress)} + prometheusService := metrics.NewPrometheusService(a.log, prometheusConfig) + a.appServices = append(a.appServices, prometheusService) + go prometheusService.Start() +} + +func (a *App) stopAppServices() { + ctx, cancel := context.WithTimeout(context.Background(), defaultShutdownTimeout) + defer cancel() + + for _, svc := range a.appServices { + svc.ShutDown(ctx) + } +} diff --git a/cmd/s3-lifecycler/logger.go b/cmd/s3-lifecycler/logger.go new file mode 100644 index 0000000..af5b4f8 --- /dev/null +++ b/cmd/s3-lifecycler/logger.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/zapjournald" + "github.com/spf13/viper" + "github.com/ssgreg/journald" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const ( + destinationStdout string = "stdout" + destinationJournald string = "journald" +) + +func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { + lvl, err := getLogLevel(v.GetString(cfgLoggerLevel)) + if err != nil { + panic(err) + } + + dest := v.GetString(cfgLoggerDestination) + + if dest == destinationStdout { + return newStdoutLogger(lvl) + } + + if dest == destinationJournald { + return newJournaldLogger(lvl) + } + + panic(fmt.Sprintf("wrong destination for logger: %s", dest)) +} + +func newStdoutLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { + c := zap.NewProductionConfig() + c.Level = zap.NewAtomicLevelAt(lvl) + c.Encoding = "console" + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + + l, err := c.Build( + zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), + ) + if err != nil { + panic(fmt.Sprintf("build zap logger instance: %v", err)) + } + + return l, c.Level +} + +func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { + c := zap.NewProductionConfig() + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + c.Level = zap.NewAtomicLevelAt(lvl) + + // We can use NewJSONEncoder instead if, say, frontend + // would like to access journald logs and parse them easily. + encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) + + core := zapjournald.NewCore(c.Level, encoder, &journald.Journal{}, zapjournald.SyslogFields) + coreWithContext := core.With([]zapcore.Field{ + zapjournald.SyslogFacility(zapjournald.LogDaemon), + zapjournald.SyslogIdentifier(), + zapjournald.SyslogPid(), + }) + l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) + return l, c.Level +} + +func getLogLevel(lvlStr string) (zapcore.Level, error) { + var lvl zapcore.Level + err := lvl.UnmarshalText([]byte(lvlStr)) + if err != nil { + return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ + "value should be one of %v", lvlStr, err, [...]zapcore.Level{ + zapcore.DebugLevel, + zapcore.InfoLevel, + zapcore.WarnLevel, + zapcore.ErrorLevel, + zapcore.DPanicLevel, + zapcore.PanicLevel, + zapcore.FatalLevel, + }) + } + return lvl, nil +} diff --git a/cmd/s3-lifecycler/main.go b/cmd/s3-lifecycler/main.go new file mode 100644 index 0000000..11c487d --- /dev/null +++ b/cmd/s3-lifecycler/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "context" + "os/signal" + "syscall" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + cfg := settings() + log, level := pickLogger(cfg) + + app := newApp(cfg, log, level) + go app.Serve(ctx) + app.Wait() +} diff --git a/cmd/s3-lifecycler/misc.go b/cmd/s3-lifecycler/misc.go new file mode 100644 index 0000000..0e57789 --- /dev/null +++ b/cmd/s3-lifecycler/misc.go @@ -0,0 +1,10 @@ +package main + +// Prefix is a prefix used for environment variables containing auth +// configuration. +const Prefix = "S3_LIFECYCLER" + +var ( + // Version is the FrostFS S3 Lifecycler service version. + Version = "dev" +) diff --git a/cmd/s3-lifecycler/settings.go b/cmd/s3-lifecycler/settings.go new file mode 100644 index 0000000..2d8db51 --- /dev/null +++ b/cmd/s3-lifecycler/settings.go @@ -0,0 +1,161 @@ +package main + +import ( + "fmt" + "os" + "path" + "runtime" + "strings" + "time" + + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +const ( + cfgPrometheusEnabled = "prometheus.enabled" + cfgPrometheusAddress = "prometheus.address" + cfgPprofEnabled = "pprof.enabled" + cfgPprofAddress = "pprof.address" + + // Logger. + cfgLoggerLevel = "logger.level" + cfgLoggerDestination = "logger.destination" + + // Command line args. + cmdHelp = "help" + cmdVersion = "version" + cmdConfig = "config" + cmdConfigDir = "config-dir" +) + +const ( + defaultShutdownTimeout = 15 * time.Second + componentName = "frostfs-s3-lifecycler" +) + +func settings() *viper.Viper { + v := viper.New() + v.AutomaticEnv() + v.SetEnvPrefix(Prefix) + v.AllowEmptyEnv(true) + v.SetConfigType("yaml") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + + // flags setup: + flags := pflag.NewFlagSet("commandline", pflag.ExitOnError) + flags.SetOutput(os.Stdout) + flags.SortFlags = false + + help := flags.BoolP(cmdHelp, "h", false, "show help") + version := flags.BoolP(cmdVersion, "v", false, "show version") + + flags.StringArray(cmdConfig, nil, "config paths") + flags.String(cmdConfigDir, "", "config dir path") + + // set defaults: + + // logger: + v.SetDefault(cfgLoggerLevel, "debug") + v.SetDefault(cfgLoggerDestination, "stdout") + + // services: + v.SetDefault(cfgPrometheusEnabled, false) + v.SetDefault(cfgPprofEnabled, false) + + // Bind flags with configuration values. + if err := v.BindPFlags(flags); err != nil { + panic(err) + } + + if err := flags.Parse(os.Args); err != nil { + panic(err) + } + + switch { + case help != nil && *help: + printVersion() + + flags.PrintDefaults() + os.Exit(0) + case version != nil && *version: + printVersion() + os.Exit(0) + } + + if err := readInConfig(v); err != nil { + panic(err) + } + + return v +} + +func readInConfig(v *viper.Viper) error { + if v.IsSet(cmdConfig) { + if err := readConfig(v); err != nil { + return err + } + } + + if v.IsSet(cmdConfigDir) { + if err := readConfigDir(v); err != nil { + return err + } + } + + return nil +} + +func readConfigDir(v *viper.Viper) error { + cfgSubConfigDir := v.GetString(cmdConfigDir) + entries, err := os.ReadDir(cfgSubConfigDir) + if err != nil { + return err + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + ext := path.Ext(entry.Name()) + if ext != ".yaml" && ext != ".yml" { + continue + } + + if err = mergeConfig(v, path.Join(cfgSubConfigDir, entry.Name())); err != nil { + return err + } + } + + return nil +} + +func readConfig(v *viper.Viper) error { + for _, fileName := range v.GetStringSlice(cmdConfig) { + if err := mergeConfig(v, fileName); err != nil { + return err + } + } + return nil +} + +func mergeConfig(v *viper.Viper, fileName string) error { + cfgFile, err := os.Open(fileName) + if err != nil { + return err + } + + defer func() { + if err2 := cfgFile.Close(); err2 != nil { + panic(err2) + } + }() + + err = v.MergeConfig(cfgFile) + + return err +} + +func printVersion() { + fmt.Printf("%s\nVersion: %s\nGoVersion: %s\n", componentName, Version, runtime.Version()) +} diff --git a/config/config.env b/config/config.env new file mode 100644 index 0000000..a5b8a2d --- /dev/null +++ b/config/config.env @@ -0,0 +1,10 @@ +# Logger +S3_GW_LOGGER_LEVEL=debug +S3_GW_LOGGER_DESTINATION=stdout + +# Metrics +S3_GW_PPROF_ENABLED=false +S3_GW_PPROF_ADDRESS=localhost:8077 + +S3_GW_PROMETHEUS_ENABLED=false +S3_GW_PROMETHEUS_ADDRESS=localhost:8078 diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..03ff9f3 --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,11 @@ +logger: + level: debug # Log level. + destination: stdout # Logging destination. + +pprof: + enabled: false + address: localhost:8077 # Endpoint for service profiling + +prometheus: + enabled: false + address: localhost:8078 # Endpoint for service metrics diff --git a/config/dir/config-pprof.yaml b/config/dir/config-pprof.yaml new file mode 100644 index 0000000..3156daa --- /dev/null +++ b/config/dir/config-pprof.yaml @@ -0,0 +1,3 @@ +pprof: + enabled: false + address: localhost:8077 diff --git a/config/dir/config-prometheus.yaml b/config/dir/config-prometheus.yaml new file mode 100644 index 0000000..60b91f1 --- /dev/null +++ b/config/dir/config-prometheus.yaml @@ -0,0 +1,3 @@ +prometheus: + enabled: false + address: localhost:8078 diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..ec9aa88 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,66 @@ +# FrostFS S3 Lifecycler configuration + +This section contains detailed FrostFS S3 Lifecycler component configuration description. + +# Structure + +| Section | Description | +|--------------|-------------------------------------------------| +| no section | [General parameters](#general-section) | +| `logger` | [Logger configuration](#logger-section) | +| `pprof` | [Pprof configuration](#pprof-section) | +| `prometheus` | [Prometheus configuration](#prometheus-section) | + +### Reload on SIGHUP + +Some config values can be reloaded on SIGHUP signal. +Such parameters have special mark in tables below. + +You can send SIGHUP signal to app using the following command: + +```shell +$ kill -s SIGHUP +``` + +# `logger` section + +```yaml +logger: + level: debug + destination: stdout +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------|----------|---------------|---------------|--------------------------------------------------------------------------------------| +| `level` | `string` | yes | `info` | Logging level. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | +| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` | + +# `pprof` section + +Contains configuration for the `pprof` profiler. + +```yaml +pprof: + enabled: false + address: localhost:8077 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|-----------------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable pprof service. | +| `address` | `string` | yes | | Address that pprof service listener binds to. | + +# `prometheus` section + +Contains configuration for the `prometheus` metrics service. + +```yaml +prometheus: + enabled: false + address: localhost:8078 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|----------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable prometheus service. | +| `address` | `string` | yes | | Address that prometheus service listener binds to. | diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..94684ad --- /dev/null +++ b/go.mod @@ -0,0 +1,41 @@ +module git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler + +go 1.21 + +require ( + git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 + github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_model v0.6.1 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.19.0 + github.com/ssgreg/journald v1.0.0 + github.com/stretchr/testify v1.9.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/help.mk b/help.mk new file mode 100644 index 0000000..b5fbbc9 --- /dev/null +++ b/help.mk @@ -0,0 +1,22 @@ +.PHONY: help + +# Show this help prompt +help: + @echo ' Usage:' + @echo '' + @echo ' make ' + @echo '' + @echo ' Targets:' + @echo '' + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9.%_/-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq + +# Show help for docker/% IGNORE +help.docker/%: + $(eval TARGETS:=$(notdir all lint) ${BINS}) + @echo ' Usage:' + @echo '' + @echo ' make docker/% -- Run `make %` in Golang container' + @echo '' + @echo ' Supported docker targets:' + @echo '' + @$(foreach bin, $(TARGETS), echo ' ' $(bin);) diff --git a/internal/logs/logs.go b/internal/logs/logs.go new file mode 100644 index 0000000..2e68e08 --- /dev/null +++ b/internal/logs/logs.go @@ -0,0 +1,18 @@ +package logs + +const ( + ApplicationStarted = "application started" + ApplicationStopped = "application stopped" + StoppingApplication = "stopping application" + ServiceIsRunning = "service is running" + ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" + ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" + ShuttingDownService = "shutting down service" + CantGracefullyShutDownService = "can't gracefully shut down service, force stop" + CantShutDownService = "can't shut down service" + SIGHUPConfigReloadStarted = "SIGHUP config reload started" + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" + FailedToReloadConfig = "failed to reload config" + LogLevelWontBeUpdated = "log level won't be updated" + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" +) diff --git a/internal/metrics/desc.go b/internal/metrics/desc.go new file mode 100644 index 0000000..4f33014 --- /dev/null +++ b/internal/metrics/desc.go @@ -0,0 +1,99 @@ +package metrics + +import ( + "encoding/json" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +var appMetricsDesc = map[string]map[string]Description{ + stateSubsystem: { + healthMetric: Description{ + Type: dto.MetricType_GAUGE, + Namespace: namespace, + Subsystem: stateSubsystem, + Name: healthMetric, + Help: "FrostFS S3 Lifecycler state", + }, + versionInfoMetric: Description{ + Type: dto.MetricType_GAUGE, + Namespace: namespace, + Subsystem: stateSubsystem, + Name: versionInfoMetric, + Help: "Version of current FrostFS S3 Lifecycler instance", + VariableLabels: []string{"version"}, + }, + }, +} + +type Description struct { + Type dto.MetricType + Namespace string + Subsystem string + Name string + Help string + ConstantLabels prometheus.Labels + VariableLabels []string +} + +func (d *Description) MarshalJSON() ([]byte, error) { + return json.Marshal(&struct { + Type string `json:"type"` + FQName string `json:"name"` + Help string `json:"help"` + ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"` + VariableLabels []string `json:"variable_labels,omitempty"` + }{ + Type: d.Type.String(), + FQName: d.BuildFQName(), + Help: d.Help, + ConstantLabels: d.ConstantLabels, + VariableLabels: d.VariableLabels, + }) +} + +func (d *Description) BuildFQName() string { + return prometheus.BuildFQName(d.Namespace, d.Subsystem, d.Name) +} + +// DescribeAll returns descriptions for metrics. +func DescribeAll() []Description { + var list []Description + for _, m := range appMetricsDesc { + for _, description := range m { + list = append(list, description) + } + } + + return list +} + +func newOpts(description Description) prometheus.Opts { + return prometheus.Opts{ + Namespace: description.Namespace, + Subsystem: description.Subsystem, + Name: description.Name, + Help: description.Help, + ConstLabels: description.ConstantLabels, + } +} + +func mustNewGauge(description Description) prometheus.Gauge { + if description.Type != dto.MetricType_GAUGE { + panic("invalid metric type") + } + return prometheus.NewGauge( + prometheus.GaugeOpts(newOpts(description)), + ) +} + +func mustNewGaugeVec(description Description) *prometheus.GaugeVec { + if description.Type != dto.MetricType_GAUGE { + panic("invalid metric type") + } + return prometheus.NewGaugeVec( + prometheus.GaugeOpts(newOpts(description)), + description.VariableLabels, + ) +} diff --git a/internal/metrics/desc_test.go b/internal/metrics/desc_test.go new file mode 100644 index 0000000..dda6c2a --- /dev/null +++ b/internal/metrics/desc_test.go @@ -0,0 +1,27 @@ +//go:build dump_metrics + +package metrics + +import ( + "encoding/json" + "flag" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +var metricsPath = flag.String("out", "", "File to export Frostfs S3 lifecycler metrics to.") + +func TestDescribeAll(t *testing.T) { + flag.Parse() + + require.NotEmpty(t, metricsPath, "flag 'out' must be provided to dump metrics description") + + desc := DescribeAll() + data, err := json.Marshal(desc) + require.NoError(t, err) + + err = os.WriteFile(*metricsPath, data, 0644) + require.NoError(t, err) +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..9474d8b --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,76 @@ +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" +) + +type ( + // AppMetrics is a metrics container for all app specific data. + AppMetrics struct { + stateMetrics + } + + // stateMetrics are metrics of application state. + stateMetrics struct { + healthCheck prometheus.Gauge + versionInfo *prometheus.GaugeVec + } +) + +const ( + namespace = "frostfs_s3_lifecycler" + stateSubsystem = "state" + + healthMetric = "health" + versionInfoMetric = "version_info" +) + +func (m stateMetrics) register() { + prometheus.MustRegister(m.healthCheck) +} + +func (m stateMetrics) SetHealth(s int32) { + m.healthCheck.Set(float64(s)) +} + +func (m stateMetrics) SetVersion(ver string) { + m.versionInfo.WithLabelValues(ver).Set(1) +} + +// NewAppMetrics creates an instance of application. +func NewAppMetrics() *AppMetrics { + stateMetric := newStateMetrics() + stateMetric.register() + + return &AppMetrics{ + stateMetrics: *stateMetric, + } +} + +func newStateMetrics() *stateMetrics { + return &stateMetrics{ + healthCheck: mustNewGauge(appMetricsDesc[stateSubsystem][healthMetric]), + versionInfo: mustNewGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]), + } +} + +// NewPrometheusService creates a new service for gathering prometheus metrics. +func NewPrometheusService(log *zap.Logger, cfg Config) *Service { + if log == nil { + return nil + } + + return &Service{ + Server: &http.Server{ + Addr: cfg.Address, + Handler: promhttp.Handler(), + }, + enabled: cfg.Enabled, + serviceType: "Prometheus", + log: log.With(zap.String("service", "Prometheus")), + } +} diff --git a/internal/metrics/profiler.go b/internal/metrics/profiler.go new file mode 100644 index 0000000..4719a69 --- /dev/null +++ b/internal/metrics/profiler.go @@ -0,0 +1,33 @@ +package metrics + +import ( + "net/http" + "net/http/pprof" + + "go.uber.org/zap" +) + +// NewPprofService creates a new service for gathering pprof metrics. +func NewPprofService(l *zap.Logger, cfg Config) *Service { + handler := http.NewServeMux() + handler.HandleFunc("/debug/pprof/", pprof.Index) + handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + handler.HandleFunc("/debug/pprof/profile", pprof.Profile) + handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + handler.HandleFunc("/debug/pprof/trace", pprof.Trace) + + // Manually add support for paths linked to by index page at /debug/pprof/ + for _, item := range []string{"allocs", "block", "heap", "goroutine", "mutex", "threadcreate"} { + handler.Handle("/debug/pprof/"+item, pprof.Handler(item)) + } + + return &Service{ + Server: &http.Server{ + Addr: cfg.Address, + Handler: handler, + }, + enabled: cfg.Enabled, + serviceType: "Pprof", + log: l.With(zap.String("service", "Pprof")), + } +} diff --git a/internal/metrics/service.go b/internal/metrics/service.go new file mode 100644 index 0000000..f347378 --- /dev/null +++ b/internal/metrics/service.go @@ -0,0 +1,49 @@ +package metrics + +import ( + "context" + "net/http" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" + "go.uber.org/zap" +) + +// Service serves metrics. +type Service struct { + *http.Server + enabled bool + log *zap.Logger + serviceType string +} + +// Config is a params to configure service. +type Config struct { + Address string + Enabled bool +} + +// Start runs http service with the exposed endpoint on the configured port. +func (ms *Service) Start() { + if ms.enabled { + // nolint: truecloudlab-linters + ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr)) + err := ms.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort) + } + } else { + ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled) + } +} + +// ShutDown stops the service. +func (ms *Service) ShutDown(ctx context.Context) { + ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr)) + err := ms.Shutdown(ctx) + if err != nil { + ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err)) + if err = ms.Close(); err != nil { + ms.log.Panic(logs.CantShutDownService, zap.Error(err)) + } + } +}