[#1] Add basic repository structure
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
parent
27189a38bb
commit
4e71fbeba6
30 changed files with 1370 additions and 0 deletions
20
.docker/Dockerfile
Normal file
20
.docker/Dockerfile
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
FROM golang:1.21 as builder
|
||||||
|
|
||||||
|
ARG BUILD=now
|
||||||
|
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler
|
||||||
|
ARG VERSION=dev
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
COPY . /src
|
||||||
|
|
||||||
|
RUN make
|
||||||
|
|
||||||
|
# Executable image
|
||||||
|
FROM alpine AS frostfs-s3-lifecycler
|
||||||
|
RUN apk add --no-cache bash ca-certificates
|
||||||
|
|
||||||
|
WORKDIR /
|
||||||
|
|
||||||
|
COPY --from=builder /src/bin/frostfs-s3-lifecycler /bin/frostfs-s3-lifecycler
|
||||||
|
|
||||||
|
ENTRYPOINT ["/bin/frostfs-s3-lifecycler"]
|
8
.docker/Dockerfile.dirty
Normal file
8
.docker/Dockerfile.dirty
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
FROM alpine AS frostfs-s3-lifecycler
|
||||||
|
RUN apk add --no-cache bash ca-certificates
|
||||||
|
|
||||||
|
WORKDIR /
|
||||||
|
|
||||||
|
COPY /bin/frostfs-s3-lifecycler /bin/frostfs-s3-lifecycler
|
||||||
|
|
||||||
|
ENTRYPOINT ["/bin/frostfs-s3-lifecycler"]
|
3
.dockerignore
Normal file
3
.dockerignore
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
.git
|
||||||
|
.cache
|
||||||
|
.github
|
7
.gitignore
vendored
Normal file
7
.gitignore
vendored
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
.idea
|
||||||
|
.cache
|
||||||
|
bin
|
||||||
|
temp
|
||||||
|
/plugins/
|
||||||
|
/vendor/
|
||||||
|
metrics-dump.json
|
11
.gitlint
Normal file
11
.gitlint
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
[general]
|
||||||
|
fail-without-commits=True
|
||||||
|
regex-style-search=True
|
||||||
|
contrib=CC1
|
||||||
|
|
||||||
|
[title-match-regex]
|
||||||
|
regex=^\[\#[0-9Xx]+\]\s
|
||||||
|
|
||||||
|
[ignore-by-title]
|
||||||
|
regex=^Release(.*)
|
||||||
|
ignore=title-match-regex
|
67
.golangci.yml
Normal file
67
.golangci.yml
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
# This file contains all available configuration options
|
||||||
|
# with their default values.
|
||||||
|
|
||||||
|
# options for analysis running
|
||||||
|
run:
|
||||||
|
# timeout for analysis, e.g. 30s, 5m, default is 1m
|
||||||
|
timeout: 15m
|
||||||
|
|
||||||
|
# include test files or not, default is true
|
||||||
|
tests: true
|
||||||
|
|
||||||
|
# output configuration options
|
||||||
|
output:
|
||||||
|
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
|
||||||
|
format: tab
|
||||||
|
|
||||||
|
# all available settings of specific linters
|
||||||
|
linters-settings:
|
||||||
|
exhaustive:
|
||||||
|
# indicates that switch statements are to be considered exhaustive if a
|
||||||
|
# 'default' case is present, even if all enum members aren't listed in the
|
||||||
|
# switch
|
||||||
|
default-signifies-exhaustive: true
|
||||||
|
govet:
|
||||||
|
# report about shadowed variables
|
||||||
|
check-shadowing: false
|
||||||
|
custom:
|
||||||
|
truecloudlab-linters:
|
||||||
|
path: bin/external_linters.so
|
||||||
|
original-url: git.frostfs.info/TrueCloudLab/linters.git
|
||||||
|
settings:
|
||||||
|
noliteral:
|
||||||
|
enable: true
|
||||||
|
target-methods: ["Fatal"]
|
||||||
|
disable-packages: []
|
||||||
|
constants-package: "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs"
|
||||||
|
|
||||||
|
linters:
|
||||||
|
enable:
|
||||||
|
# mandatory linters
|
||||||
|
- govet
|
||||||
|
- revive
|
||||||
|
|
||||||
|
# some default golangci-lint linters
|
||||||
|
- errcheck
|
||||||
|
- gosimple
|
||||||
|
- ineffassign
|
||||||
|
- staticcheck
|
||||||
|
- typecheck
|
||||||
|
- unused
|
||||||
|
|
||||||
|
# extra linters
|
||||||
|
- exhaustive
|
||||||
|
- godot
|
||||||
|
- gofmt
|
||||||
|
- whitespace
|
||||||
|
- goimports
|
||||||
|
- truecloudlab-linters
|
||||||
|
disable-all: true
|
||||||
|
fast: false
|
||||||
|
|
||||||
|
issues:
|
||||||
|
include:
|
||||||
|
- EXC0002 # should have a comment
|
||||||
|
- EXC0003 # test/Test ... consider calling this
|
||||||
|
- EXC0004 # govet
|
||||||
|
- EXC0005 # C-style breaks
|
52
.pre-commit-config.yaml
Normal file
52
.pre-commit-config.yaml
Normal file
|
@ -0,0 +1,52 @@
|
||||||
|
ci:
|
||||||
|
autofix_prs: false
|
||||||
|
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/jorisroovers/gitlint
|
||||||
|
rev: v0.19.1
|
||||||
|
hooks:
|
||||||
|
- id: gitlint
|
||||||
|
stages: [commit-msg]
|
||||||
|
- id: gitlint-ci
|
||||||
|
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v4.4.0
|
||||||
|
hooks:
|
||||||
|
- id: check-added-large-files
|
||||||
|
- id: check-case-conflict
|
||||||
|
- id: check-executables-have-shebangs
|
||||||
|
- id: check-shebang-scripts-are-executable
|
||||||
|
- id: check-merge-conflict
|
||||||
|
- id: check-json
|
||||||
|
- id: check-xml
|
||||||
|
- id: check-yaml
|
||||||
|
- id: trailing-whitespace
|
||||||
|
args: [--markdown-linebreak-ext=md]
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
exclude: ".key$"
|
||||||
|
|
||||||
|
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||||
|
rev: v0.9.0.2
|
||||||
|
hooks:
|
||||||
|
- id: shellcheck
|
||||||
|
|
||||||
|
- repo: local
|
||||||
|
hooks:
|
||||||
|
- id: make-lint-install
|
||||||
|
name: install linters
|
||||||
|
entry: make lint-install
|
||||||
|
language: system
|
||||||
|
pass_filenames: false
|
||||||
|
|
||||||
|
- id: make-lint
|
||||||
|
name: run linters
|
||||||
|
entry: make lint
|
||||||
|
language: system
|
||||||
|
pass_filenames: false
|
||||||
|
|
||||||
|
- id: go-unit-tests
|
||||||
|
name: go unit tests
|
||||||
|
entry: make test
|
||||||
|
pass_filenames: false
|
||||||
|
types: [go]
|
||||||
|
language: system
|
5
CHANGELOG.md
Normal file
5
CHANGELOG.md
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
# Changelog
|
||||||
|
|
||||||
|
This document outlines major changes between releases.
|
||||||
|
|
||||||
|
## [Unreleased]
|
156
CONTRIBUTING.md
Normal file
156
CONTRIBUTING.md
Normal file
|
@ -0,0 +1,156 @@
|
||||||
|
# Contribution guide
|
||||||
|
|
||||||
|
First, thank you for contributing! We love and encourage pull requests from
|
||||||
|
everyone. Please follow the guidelines:
|
||||||
|
|
||||||
|
- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/issues) and
|
||||||
|
[pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/pulls) for existing
|
||||||
|
discussions.
|
||||||
|
|
||||||
|
- Open an issue first, to discuss a new feature or enhancement.
|
||||||
|
|
||||||
|
- Write tests and make sure the test suite passes locally and on CI.
|
||||||
|
|
||||||
|
- Open a pull request and reference the relevant issue(s).
|
||||||
|
|
||||||
|
- Make sure your commits are logically separated and have good comments
|
||||||
|
explaining the details of your change.
|
||||||
|
|
||||||
|
- After receiving a feedback, amend your commits or add new ones as
|
||||||
|
appropriate.
|
||||||
|
|
||||||
|
- **Have fun!**
|
||||||
|
|
||||||
|
## Development Workflow
|
||||||
|
|
||||||
|
Start by forking the `frostfs-s3-lifecycler` repository, make changes in a branch and then
|
||||||
|
send a pull request. We encourage pull requests to discuss code changes. Here
|
||||||
|
are the steps in details:
|
||||||
|
|
||||||
|
### Set up your git repository
|
||||||
|
Fork [FrostFS S3 Gateway
|
||||||
|
upstream](https://git.frostfs.info/repo/fork/15) source repository
|
||||||
|
to your own personal repository. Copy the URL of your fork (you will need it for
|
||||||
|
the `git clone` command below).
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ git clone https://git.frostfs.info/<username>/frostfs-s3-lifecycler.git
|
||||||
|
```
|
||||||
|
|
||||||
|
### Set up git remote as ``upstream``
|
||||||
|
```sh
|
||||||
|
$ cd frostfs-s3-lifecycler
|
||||||
|
$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler.git
|
||||||
|
$ git fetch upstream
|
||||||
|
$ git merge upstream/master
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Create your feature branch
|
||||||
|
Before making code changes, make sure you create a separate branch for these
|
||||||
|
changes. Maybe you will find it convenient to name a branch in
|
||||||
|
`<type>/<Issue>-<changes_topic>` format.
|
||||||
|
|
||||||
|
```
|
||||||
|
$ git checkout -b feature/123-something_awesome
|
||||||
|
```
|
||||||
|
|
||||||
|
### Test your changes
|
||||||
|
After your code changes, make sure
|
||||||
|
|
||||||
|
- To add test cases for the new code.
|
||||||
|
- To run `make lint`
|
||||||
|
- To squash your commits into a single commit or a series of logically separated
|
||||||
|
commits with `git rebase -i`. It's okay to force update your pull request.
|
||||||
|
- To run `make test` and `make all` successfully.
|
||||||
|
|
||||||
|
### Commit changes
|
||||||
|
After verification, commit your changes. There is a [great
|
||||||
|
post](https://chris.beams.io/posts/git-commit/) on how to write useful commit
|
||||||
|
messages. Try following this template:
|
||||||
|
|
||||||
|
```
|
||||||
|
[#Issue] <component> Summary
|
||||||
|
|
||||||
|
Description
|
||||||
|
|
||||||
|
<Macros>
|
||||||
|
|
||||||
|
<Sign-Off>
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
$ git commit -ams '[#123] Add some feature'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Push to the branch
|
||||||
|
Push your locally committed changes to the remote origin (your fork)
|
||||||
|
```
|
||||||
|
$ git push origin feature/123-something_awesome
|
||||||
|
```
|
||||||
|
|
||||||
|
### Create a Pull Request
|
||||||
|
Pull requests can be created via Forgejo. Refer to [this
|
||||||
|
document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for
|
||||||
|
detailed steps on how to create a pull request. After a Pull Request gets peer
|
||||||
|
reviewed and approved, it will be merged.
|
||||||
|
|
||||||
|
## DCO Sign off
|
||||||
|
|
||||||
|
All authors to the project retain copyright to their work. However, to ensure
|
||||||
|
that they are only submitting work that they have rights to, we require
|
||||||
|
everyone to acknowledge this by signing their work.
|
||||||
|
|
||||||
|
Any copyright notices in this repository should specify the authors as "the
|
||||||
|
contributors".
|
||||||
|
|
||||||
|
To sign your work, just add a line like this at the end of your commit message:
|
||||||
|
|
||||||
|
```
|
||||||
|
Signed-off-by: Samii Sakisaka <samii@frostfs.info>
|
||||||
|
```
|
||||||
|
|
||||||
|
This can be easily done with the `--signoff` option to `git commit`.
|
||||||
|
|
||||||
|
By doing this you state that you can certify the following (from [The Developer
|
||||||
|
Certificate of Origin](https://developercertificate.org/)):
|
||||||
|
|
||||||
|
```
|
||||||
|
Developer Certificate of Origin
|
||||||
|
Version 1.1
|
||||||
|
|
||||||
|
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
|
||||||
|
1 Letterman Drive
|
||||||
|
Suite D4700
|
||||||
|
San Francisco, CA, 94129
|
||||||
|
|
||||||
|
Everyone is permitted to copy and distribute verbatim copies of this
|
||||||
|
license document, but changing it is not allowed.
|
||||||
|
|
||||||
|
|
||||||
|
Developer's Certificate of Origin 1.1
|
||||||
|
|
||||||
|
By making a contribution to this project, I certify that:
|
||||||
|
|
||||||
|
(a) The contribution was created in whole or in part by me and I
|
||||||
|
have the right to submit it under the open source license
|
||||||
|
indicated in the file; or
|
||||||
|
|
||||||
|
(b) The contribution is based upon previous work that, to the best
|
||||||
|
of my knowledge, is covered under an appropriate open source
|
||||||
|
license and I have the right under that license to submit that
|
||||||
|
work with modifications, whether created in whole or in part
|
||||||
|
by me, under the same open source license (unless I am
|
||||||
|
permitted to submit under a different license), as indicated
|
||||||
|
in the file; or
|
||||||
|
|
||||||
|
(c) The contribution was provided directly to me by some other
|
||||||
|
person who certified (a), (b) or (c) and I have not modified
|
||||||
|
it.
|
||||||
|
|
||||||
|
(d) I understand and agree that this project and the contribution
|
||||||
|
are public and that a record of the contribution (including all
|
||||||
|
personal information I submit with it, including my sign-off) is
|
||||||
|
maintained indefinitely and may be redistributed consistent with
|
||||||
|
this project or the open source license(s) involved.
|
||||||
|
```
|
11
CREDITS.md
Normal file
11
CREDITS.md
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
# Credits
|
||||||
|
|
||||||
|
In alphabetical order:
|
||||||
|
|
||||||
|
- Denis Kirillov (@dkirillov)
|
||||||
|
|
||||||
|
# Contributors
|
||||||
|
|
||||||
|
In chronological order:
|
||||||
|
|
||||||
|
- Denis Kirillov (@dkirillov)
|
166
Makefile
Executable file
166
Makefile
Executable file
|
@ -0,0 +1,166 @@
|
||||||
|
#!/usr/bin/make -f
|
||||||
|
|
||||||
|
REPO ?= $(shell go list -m)
|
||||||
|
VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
||||||
|
GO_VERSION ?= 1.21
|
||||||
|
LINT_VERSION ?= 1.56.1
|
||||||
|
TRUECLOUDLAB_LINT_VERSION ?= 0.0.5
|
||||||
|
BINDIR = bin
|
||||||
|
|
||||||
|
METRICS_DUMP_OUT ?= ./metrics-dump.json
|
||||||
|
|
||||||
|
CMDS = $(notdir $(basename $(wildcard cmd/*)))
|
||||||
|
BINS = $(addprefix $(BINDIR)/, $(CMDS))
|
||||||
|
|
||||||
|
# Variables for docker
|
||||||
|
REPO_BASENAME = $(shell basename `go list -m`)
|
||||||
|
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||||
|
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||||
|
|
||||||
|
OUTPUT_LINT_DIR ?= $(shell pwd)/bin
|
||||||
|
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
|
||||||
|
TMP_DIR := .cache
|
||||||
|
|
||||||
|
# Make all binaries
|
||||||
|
.PHONY: all
|
||||||
|
all: $(BINS)
|
||||||
|
|
||||||
|
.PHONY: $(BINS)
|
||||||
|
$(BINS): $(BINDIR) dep fmts
|
||||||
|
@echo "⇒ Build $@"
|
||||||
|
CGO_ENABLED=0 \
|
||||||
|
go build -v -trimpath \
|
||||||
|
-ldflags "-X main.Version=$(VERSION)" \
|
||||||
|
-o bin/frostfs-s3-lifecycler ./cmd/$(notdir $@)
|
||||||
|
|
||||||
|
.PHONY: $(BINDIR)
|
||||||
|
$(BINDIR):
|
||||||
|
@echo "⇒ Ensure dir: $@"
|
||||||
|
@mkdir -p $@
|
||||||
|
|
||||||
|
# Pull go dependencies
|
||||||
|
.PHONY: dep
|
||||||
|
dep:
|
||||||
|
@printf "⇒ Download requirements: "
|
||||||
|
@CGO_ENABLED=0 \
|
||||||
|
go mod download && echo OK
|
||||||
|
@printf "⇒ Tidy requirements: "
|
||||||
|
@CGO_ENABLED=0 \
|
||||||
|
go mod tidy -v && echo OK
|
||||||
|
|
||||||
|
.PHONY: image
|
||||||
|
image:
|
||||||
|
@echo "⇒ Build FrostFS S3 Lifecycler docker image "
|
||||||
|
@docker build \
|
||||||
|
--build-arg REPO=$(REPO) \
|
||||||
|
--build-arg VERSION=$(VERSION) \
|
||||||
|
--rm \
|
||||||
|
-f .docker/Dockerfile \
|
||||||
|
-t $(HUB_IMAGE):$(HUB_TAG) .
|
||||||
|
|
||||||
|
.PHONY: image-push
|
||||||
|
image-push:
|
||||||
|
@echo "⇒ Publish image"
|
||||||
|
@docker push $(HUB_IMAGE):$(HUB_TAG)
|
||||||
|
|
||||||
|
.PHONY: dirty-image
|
||||||
|
dirty-image:
|
||||||
|
@echo "⇒ Build FrostFS S3 Lifecycler dirty docker image "
|
||||||
|
@docker build \
|
||||||
|
--build-arg REPO=$(REPO) \
|
||||||
|
--build-arg VERSION=$(VERSION) \
|
||||||
|
--rm \
|
||||||
|
-f .docker/Dockerfile.dirty \
|
||||||
|
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
|
||||||
|
|
||||||
|
.PHONY: docker/
|
||||||
|
docker/%:
|
||||||
|
$(if $(filter $*,all $(BINS)), \
|
||||||
|
@echo "=> Running 'make $*' in clean Docker environment" && \
|
||||||
|
docker run --rm -t \
|
||||||
|
-v `pwd`:/src \
|
||||||
|
-w /src \
|
||||||
|
-u `stat -c "%u:%g" .` \
|
||||||
|
--env HOME=/src \
|
||||||
|
golang:$(GO_VERSION) make $*,\
|
||||||
|
@echo "supported docker targets: all $(BINS) lint")
|
||||||
|
|
||||||
|
# Run tests
|
||||||
|
.PHONY: test
|
||||||
|
test:
|
||||||
|
@go test ./... -cover
|
||||||
|
|
||||||
|
# Run tests with race detection and produce coverage output
|
||||||
|
.PHONY: cover
|
||||||
|
cover:
|
||||||
|
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
|
||||||
|
@go tool cover -html=coverage.txt -o coverage.html
|
||||||
|
|
||||||
|
# Run all code formatters
|
||||||
|
.PHONY: fmts
|
||||||
|
fmts: fmt imports
|
||||||
|
|
||||||
|
# Reformat code
|
||||||
|
.PHONY: fmt
|
||||||
|
fmt:
|
||||||
|
@echo "⇒ Processing gofmt check"
|
||||||
|
@GO111MODULE=on gofmt -s -w ./
|
||||||
|
|
||||||
|
# Reformat imports
|
||||||
|
.PHONY: imports
|
||||||
|
imports:
|
||||||
|
@echo "⇒ Processing goimports check"
|
||||||
|
@GO111MODULE=on goimports -w ./
|
||||||
|
|
||||||
|
# Install linters
|
||||||
|
.PHONY: lint-install
|
||||||
|
lint-install:
|
||||||
|
@if [ ! -d "$(LINT_DIR)" ]; then \
|
||||||
|
mkdir -p $(TMP_DIR); \
|
||||||
|
rm -rf $(TMP_DIR)/linters; \
|
||||||
|
git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters; \
|
||||||
|
make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR); \
|
||||||
|
rm -rf $(TMP_DIR)/linters; \
|
||||||
|
rmdir $(TMP_DIR) 2>/dev/null || true; \
|
||||||
|
CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION); \
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run linters
|
||||||
|
.PHONY: lint
|
||||||
|
lint: lint-install
|
||||||
|
$(LINT_DIR)/golangci-lint --timeout=5m run
|
||||||
|
|
||||||
|
# Run linters in Docker
|
||||||
|
.PHONY: docker/lint
|
||||||
|
docker/lint:
|
||||||
|
docker run --rm -it \
|
||||||
|
-v `pwd`:/src \
|
||||||
|
-u `stat -c "%u:%g" .` \
|
||||||
|
--env HOME=/src \
|
||||||
|
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
||||||
|
|
||||||
|
# Activate pre-commit hooks
|
||||||
|
.PHONY: pre-commit
|
||||||
|
pre-commit:
|
||||||
|
pre-commit install -t pre-commit -t commit-msg
|
||||||
|
|
||||||
|
# Deactivate pre-commit hooks
|
||||||
|
.PHONY: unpre-commit
|
||||||
|
unpre-commit:
|
||||||
|
pre-commit uninstall -t pre-commit -t commit-msg
|
||||||
|
|
||||||
|
.PHONY: clean
|
||||||
|
clean:
|
||||||
|
@rm -rf $(DIRS)
|
||||||
|
|
||||||
|
# Show current version
|
||||||
|
.PHONY: version
|
||||||
|
version:
|
||||||
|
@echo $(VERSION)
|
||||||
|
|
||||||
|
# Dump metrics (use METRICS_DUMP_OUT variable to override default out file './metrics-dump.json')
|
||||||
|
.PHONY: dump-metrics
|
||||||
|
dump-metrics:
|
||||||
|
@go test ./internal/metrics -run TestDescribeAll --tags=dump_metrics --out=$(abspath $(METRICS_DUMP_OUT))
|
||||||
|
|
||||||
|
include help.mk
|
1
VERSION
Normal file
1
VERSION
Normal file
|
@ -0,0 +1 @@
|
||||||
|
v0.1.0
|
127
cmd/s3-lifecycler/app.go
Normal file
127
cmd/s3-lifecycler/app.go
Normal file
|
@ -0,0 +1,127 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/metrics"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
App struct {
|
||||||
|
log *zap.Logger
|
||||||
|
logLevel zap.AtomicLevel
|
||||||
|
cfg *viper.Viper
|
||||||
|
done chan struct{}
|
||||||
|
appServices []*metrics.Service
|
||||||
|
appMetrics *metrics.AppMetrics
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
HealthStatusUndefined int32 = 0
|
||||||
|
HealthStatusStarting int32 = 1
|
||||||
|
HealthStatusReady int32 = 2
|
||||||
|
HealthStatusShuttingDown int32 = 3
|
||||||
|
)
|
||||||
|
|
||||||
|
func newApp(cfg *viper.Viper, log *zap.Logger, level zap.AtomicLevel) *App {
|
||||||
|
a := &App{
|
||||||
|
log: log,
|
||||||
|
logLevel: level,
|
||||||
|
cfg: cfg,
|
||||||
|
done: make(chan struct{}),
|
||||||
|
appMetrics: metrics.NewAppMetrics(),
|
||||||
|
}
|
||||||
|
a.appMetrics.SetHealth(HealthStatusStarting)
|
||||||
|
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) Wait() {
|
||||||
|
a.log.Info(logs.ApplicationStarted,
|
||||||
|
zap.String("app_name", "frostfs-s3-lifecycler"),
|
||||||
|
zap.String("version", Version))
|
||||||
|
|
||||||
|
a.appMetrics.SetHealth(HealthStatusReady)
|
||||||
|
a.appMetrics.SetVersion(Version)
|
||||||
|
|
||||||
|
<-a.done
|
||||||
|
|
||||||
|
a.log.Info(logs.ApplicationStopped)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) Serve(ctx context.Context) {
|
||||||
|
sigs := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigs, syscall.SIGHUP)
|
||||||
|
|
||||||
|
a.startAppServices()
|
||||||
|
|
||||||
|
loop:
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
break loop
|
||||||
|
case <-sigs:
|
||||||
|
a.configReload()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
a.log.Info(logs.StoppingApplication)
|
||||||
|
|
||||||
|
a.appMetrics.SetHealth(HealthStatusShuttingDown)
|
||||||
|
a.stopAppServices()
|
||||||
|
|
||||||
|
close(a.done)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) configReload() {
|
||||||
|
a.log.Info(logs.SIGHUPConfigReloadStarted)
|
||||||
|
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
|
||||||
|
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := readInConfig(a.cfg); err != nil {
|
||||||
|
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if lvl, err := getLogLevel(a.cfg.GetString(cfgLoggerLevel)); err != nil {
|
||||||
|
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err))
|
||||||
|
} else {
|
||||||
|
a.logLevel.SetLevel(lvl)
|
||||||
|
}
|
||||||
|
|
||||||
|
a.stopAppServices()
|
||||||
|
a.startAppServices()
|
||||||
|
|
||||||
|
a.log.Info(logs.SIGHUPConfigReloadCompleted)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) startAppServices() {
|
||||||
|
a.appServices = a.appServices[:0]
|
||||||
|
|
||||||
|
pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)}
|
||||||
|
pprofService := metrics.NewPprofService(a.log, pprofConfig)
|
||||||
|
a.appServices = append(a.appServices, pprofService)
|
||||||
|
go pprofService.Start()
|
||||||
|
|
||||||
|
prometheusConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPrometheusEnabled), Address: a.cfg.GetString(cfgPrometheusAddress)}
|
||||||
|
prometheusService := metrics.NewPrometheusService(a.log, prometheusConfig)
|
||||||
|
a.appServices = append(a.appServices, prometheusService)
|
||||||
|
go prometheusService.Start()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) stopAppServices() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), defaultShutdownTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
for _, svc := range a.appServices {
|
||||||
|
svc.ShutDown(ctx)
|
||||||
|
}
|
||||||
|
}
|
88
cmd/s3-lifecycler/logger.go
Normal file
88
cmd/s3-lifecycler/logger.go
Normal file
|
@ -0,0 +1,88 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/zapjournald"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"github.com/ssgreg/journald"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"go.uber.org/zap/zapcore"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
destinationStdout string = "stdout"
|
||||||
|
destinationJournald string = "journald"
|
||||||
|
)
|
||||||
|
|
||||||
|
func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) {
|
||||||
|
lvl, err := getLogLevel(v.GetString(cfgLoggerLevel))
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dest := v.GetString(cfgLoggerDestination)
|
||||||
|
|
||||||
|
if dest == destinationStdout {
|
||||||
|
return newStdoutLogger(lvl)
|
||||||
|
}
|
||||||
|
|
||||||
|
if dest == destinationJournald {
|
||||||
|
return newJournaldLogger(lvl)
|
||||||
|
}
|
||||||
|
|
||||||
|
panic(fmt.Sprintf("wrong destination for logger: %s", dest))
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStdoutLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) {
|
||||||
|
c := zap.NewProductionConfig()
|
||||||
|
c.Level = zap.NewAtomicLevelAt(lvl)
|
||||||
|
c.Encoding = "console"
|
||||||
|
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||||
|
|
||||||
|
l, err := c.Build(
|
||||||
|
zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
panic(fmt.Sprintf("build zap logger instance: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
return l, c.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) {
|
||||||
|
c := zap.NewProductionConfig()
|
||||||
|
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||||
|
c.Level = zap.NewAtomicLevelAt(lvl)
|
||||||
|
|
||||||
|
// We can use NewJSONEncoder instead if, say, frontend
|
||||||
|
// would like to access journald logs and parse them easily.
|
||||||
|
encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields)
|
||||||
|
|
||||||
|
core := zapjournald.NewCore(c.Level, encoder, &journald.Journal{}, zapjournald.SyslogFields)
|
||||||
|
coreWithContext := core.With([]zapcore.Field{
|
||||||
|
zapjournald.SyslogFacility(zapjournald.LogDaemon),
|
||||||
|
zapjournald.SyslogIdentifier(),
|
||||||
|
zapjournald.SyslogPid(),
|
||||||
|
})
|
||||||
|
l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
|
||||||
|
return l, c.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
func getLogLevel(lvlStr string) (zapcore.Level, error) {
|
||||||
|
var lvl zapcore.Level
|
||||||
|
err := lvl.UnmarshalText([]byte(lvlStr))
|
||||||
|
if err != nil {
|
||||||
|
return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+
|
||||||
|
"value should be one of %v", lvlStr, err, [...]zapcore.Level{
|
||||||
|
zapcore.DebugLevel,
|
||||||
|
zapcore.InfoLevel,
|
||||||
|
zapcore.WarnLevel,
|
||||||
|
zapcore.ErrorLevel,
|
||||||
|
zapcore.DPanicLevel,
|
||||||
|
zapcore.PanicLevel,
|
||||||
|
zapcore.FatalLevel,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return lvl, nil
|
||||||
|
}
|
19
cmd/s3-lifecycler/main.go
Normal file
19
cmd/s3-lifecycler/main.go
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cfg := settings()
|
||||||
|
log, level := pickLogger(cfg)
|
||||||
|
|
||||||
|
app := newApp(cfg, log, level)
|
||||||
|
go app.Serve(ctx)
|
||||||
|
app.Wait()
|
||||||
|
}
|
10
cmd/s3-lifecycler/misc.go
Normal file
10
cmd/s3-lifecycler/misc.go
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
// Prefix is a prefix used for environment variables containing auth
|
||||||
|
// configuration.
|
||||||
|
const Prefix = "S3_LIFECYCLER"
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Version is the FrostFS S3 Lifecycler service version.
|
||||||
|
Version = "dev"
|
||||||
|
)
|
161
cmd/s3-lifecycler/settings.go
Normal file
161
cmd/s3-lifecycler/settings.go
Normal file
|
@ -0,0 +1,161 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/pflag"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
cfgPrometheusEnabled = "prometheus.enabled"
|
||||||
|
cfgPrometheusAddress = "prometheus.address"
|
||||||
|
cfgPprofEnabled = "pprof.enabled"
|
||||||
|
cfgPprofAddress = "pprof.address"
|
||||||
|
|
||||||
|
// Logger.
|
||||||
|
cfgLoggerLevel = "logger.level"
|
||||||
|
cfgLoggerDestination = "logger.destination"
|
||||||
|
|
||||||
|
// Command line args.
|
||||||
|
cmdHelp = "help"
|
||||||
|
cmdVersion = "version"
|
||||||
|
cmdConfig = "config"
|
||||||
|
cmdConfigDir = "config-dir"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultShutdownTimeout = 15 * time.Second
|
||||||
|
componentName = "frostfs-s3-lifecycler"
|
||||||
|
)
|
||||||
|
|
||||||
|
func settings() *viper.Viper {
|
||||||
|
v := viper.New()
|
||||||
|
v.AutomaticEnv()
|
||||||
|
v.SetEnvPrefix(Prefix)
|
||||||
|
v.AllowEmptyEnv(true)
|
||||||
|
v.SetConfigType("yaml")
|
||||||
|
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
||||||
|
|
||||||
|
// flags setup:
|
||||||
|
flags := pflag.NewFlagSet("commandline", pflag.ExitOnError)
|
||||||
|
flags.SetOutput(os.Stdout)
|
||||||
|
flags.SortFlags = false
|
||||||
|
|
||||||
|
help := flags.BoolP(cmdHelp, "h", false, "show help")
|
||||||
|
version := flags.BoolP(cmdVersion, "v", false, "show version")
|
||||||
|
|
||||||
|
flags.StringArray(cmdConfig, nil, "config paths")
|
||||||
|
flags.String(cmdConfigDir, "", "config dir path")
|
||||||
|
|
||||||
|
// set defaults:
|
||||||
|
|
||||||
|
// logger:
|
||||||
|
v.SetDefault(cfgLoggerLevel, "debug")
|
||||||
|
v.SetDefault(cfgLoggerDestination, "stdout")
|
||||||
|
|
||||||
|
// services:
|
||||||
|
v.SetDefault(cfgPrometheusEnabled, false)
|
||||||
|
v.SetDefault(cfgPprofEnabled, false)
|
||||||
|
|
||||||
|
// Bind flags with configuration values.
|
||||||
|
if err := v.BindPFlags(flags); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := flags.Parse(os.Args); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case help != nil && *help:
|
||||||
|
printVersion()
|
||||||
|
|
||||||
|
flags.PrintDefaults()
|
||||||
|
os.Exit(0)
|
||||||
|
case version != nil && *version:
|
||||||
|
printVersion()
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := readInConfig(v); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func readInConfig(v *viper.Viper) error {
|
||||||
|
if v.IsSet(cmdConfig) {
|
||||||
|
if err := readConfig(v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.IsSet(cmdConfigDir) {
|
||||||
|
if err := readConfigDir(v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readConfigDir(v *viper.Viper) error {
|
||||||
|
cfgSubConfigDir := v.GetString(cmdConfigDir)
|
||||||
|
entries, err := os.ReadDir(cfgSubConfigDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ext := path.Ext(entry.Name())
|
||||||
|
if ext != ".yaml" && ext != ".yml" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = mergeConfig(v, path.Join(cfgSubConfigDir, entry.Name())); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readConfig(v *viper.Viper) error {
|
||||||
|
for _, fileName := range v.GetStringSlice(cmdConfig) {
|
||||||
|
if err := mergeConfig(v, fileName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeConfig(v *viper.Viper, fileName string) error {
|
||||||
|
cfgFile, err := os.Open(fileName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if err2 := cfgFile.Close(); err2 != nil {
|
||||||
|
panic(err2)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
err = v.MergeConfig(cfgFile)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func printVersion() {
|
||||||
|
fmt.Printf("%s\nVersion: %s\nGoVersion: %s\n", componentName, Version, runtime.Version())
|
||||||
|
}
|
10
config/config.env
Normal file
10
config/config.env
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
# Logger
|
||||||
|
S3_GW_LOGGER_LEVEL=debug
|
||||||
|
S3_GW_LOGGER_DESTINATION=stdout
|
||||||
|
|
||||||
|
# Metrics
|
||||||
|
S3_GW_PPROF_ENABLED=false
|
||||||
|
S3_GW_PPROF_ADDRESS=localhost:8077
|
||||||
|
|
||||||
|
S3_GW_PROMETHEUS_ENABLED=false
|
||||||
|
S3_GW_PROMETHEUS_ADDRESS=localhost:8078
|
11
config/config.yaml
Normal file
11
config/config.yaml
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
logger:
|
||||||
|
level: debug # Log level.
|
||||||
|
destination: stdout # Logging destination.
|
||||||
|
|
||||||
|
pprof:
|
||||||
|
enabled: false
|
||||||
|
address: localhost:8077 # Endpoint for service profiling
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
enabled: false
|
||||||
|
address: localhost:8078 # Endpoint for service metrics
|
3
config/dir/config-pprof.yaml
Normal file
3
config/dir/config-pprof.yaml
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
pprof:
|
||||||
|
enabled: false
|
||||||
|
address: localhost:8077
|
3
config/dir/config-prometheus.yaml
Normal file
3
config/dir/config-prometheus.yaml
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
prometheus:
|
||||||
|
enabled: false
|
||||||
|
address: localhost:8078
|
66
docs/configuration.md
Normal file
66
docs/configuration.md
Normal file
|
@ -0,0 +1,66 @@
|
||||||
|
# FrostFS S3 Lifecycler configuration
|
||||||
|
|
||||||
|
This section contains detailed FrostFS S3 Lifecycler component configuration description.
|
||||||
|
|
||||||
|
# Structure
|
||||||
|
|
||||||
|
| Section | Description |
|
||||||
|
|--------------|-------------------------------------------------|
|
||||||
|
| no section | [General parameters](#general-section) |
|
||||||
|
| `logger` | [Logger configuration](#logger-section) |
|
||||||
|
| `pprof` | [Pprof configuration](#pprof-section) |
|
||||||
|
| `prometheus` | [Prometheus configuration](#prometheus-section) |
|
||||||
|
|
||||||
|
### Reload on SIGHUP
|
||||||
|
|
||||||
|
Some config values can be reloaded on SIGHUP signal.
|
||||||
|
Such parameters have special mark in tables below.
|
||||||
|
|
||||||
|
You can send SIGHUP signal to app using the following command:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
$ kill -s SIGHUP <app_pid>
|
||||||
|
```
|
||||||
|
|
||||||
|
# `logger` section
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
logger:
|
||||||
|
level: debug
|
||||||
|
destination: stdout
|
||||||
|
```
|
||||||
|
|
||||||
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|
|---------------|----------|---------------|---------------|--------------------------------------------------------------------------------------|
|
||||||
|
| `level` | `string` | yes | `info` | Logging level. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. |
|
||||||
|
| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` |
|
||||||
|
|
||||||
|
# `pprof` section
|
||||||
|
|
||||||
|
Contains configuration for the `pprof` profiler.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
pprof:
|
||||||
|
enabled: false
|
||||||
|
address: localhost:8077
|
||||||
|
```
|
||||||
|
|
||||||
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|
|-----------|----------|---------------|---------------|-----------------------------------------------|
|
||||||
|
| `enabled` | `bool` | yes | `false` | Flag to enable pprof service. |
|
||||||
|
| `address` | `string` | yes | | Address that pprof service listener binds to. |
|
||||||
|
|
||||||
|
# `prometheus` section
|
||||||
|
|
||||||
|
Contains configuration for the `prometheus` metrics service.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
prometheus:
|
||||||
|
enabled: false
|
||||||
|
address: localhost:8078
|
||||||
|
```
|
||||||
|
|
||||||
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|
|-----------|----------|---------------|---------------|----------------------------------------------------|
|
||||||
|
| `enabled` | `bool` | yes | `false` | Flag to enable prometheus service. |
|
||||||
|
| `address` | `string` | yes | | Address that prometheus service listener binds to. |
|
41
go.mod
Normal file
41
go.mod
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
module git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler
|
||||||
|
|
||||||
|
go 1.21
|
||||||
|
|
||||||
|
require (
|
||||||
|
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
||||||
|
github.com/prometheus/client_golang v1.19.1
|
||||||
|
github.com/prometheus/client_model v0.6.1
|
||||||
|
github.com/spf13/pflag v1.0.5
|
||||||
|
github.com/spf13/viper v1.19.0
|
||||||
|
github.com/ssgreg/journald v1.0.0
|
||||||
|
github.com/stretchr/testify v1.9.0
|
||||||
|
go.uber.org/zap v1.27.0
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
|
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||||
|
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||||
|
github.com/magiconair/properties v1.8.7 // indirect
|
||||||
|
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||||
|
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
|
github.com/prometheus/common v0.48.0 // indirect
|
||||||
|
github.com/prometheus/procfs v0.12.0 // indirect
|
||||||
|
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||||
|
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||||
|
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||||
|
github.com/spf13/afero v1.11.0 // indirect
|
||||||
|
github.com/spf13/cast v1.6.0 // indirect
|
||||||
|
github.com/subosito/gotenv v1.6.0 // indirect
|
||||||
|
go.uber.org/multierr v1.10.0 // indirect
|
||||||
|
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
||||||
|
golang.org/x/sys v0.20.0 // indirect
|
||||||
|
golang.org/x/text v0.16.0 // indirect
|
||||||
|
google.golang.org/protobuf v1.33.0 // indirect
|
||||||
|
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
|
)
|
22
help.mk
Normal file
22
help.mk
Normal file
|
@ -0,0 +1,22 @@
|
||||||
|
.PHONY: help
|
||||||
|
|
||||||
|
# Show this help prompt
|
||||||
|
help:
|
||||||
|
@echo ' Usage:'
|
||||||
|
@echo ''
|
||||||
|
@echo ' make <target>'
|
||||||
|
@echo ''
|
||||||
|
@echo ' Targets:'
|
||||||
|
@echo ''
|
||||||
|
@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9.%_/-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq
|
||||||
|
|
||||||
|
# Show help for docker/% IGNORE
|
||||||
|
help.docker/%:
|
||||||
|
$(eval TARGETS:=$(notdir all lint) ${BINS})
|
||||||
|
@echo ' Usage:'
|
||||||
|
@echo ''
|
||||||
|
@echo ' make docker/% -- Run `make %` in Golang container'
|
||||||
|
@echo ''
|
||||||
|
@echo ' Supported docker targets:'
|
||||||
|
@echo ''
|
||||||
|
@$(foreach bin, $(TARGETS), echo ' ' $(bin);)
|
18
internal/logs/logs.go
Normal file
18
internal/logs/logs.go
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
package logs
|
||||||
|
|
||||||
|
const (
|
||||||
|
ApplicationStarted = "application started"
|
||||||
|
ApplicationStopped = "application stopped"
|
||||||
|
StoppingApplication = "stopping application"
|
||||||
|
ServiceIsRunning = "service is running"
|
||||||
|
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port"
|
||||||
|
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled"
|
||||||
|
ShuttingDownService = "shutting down service"
|
||||||
|
CantGracefullyShutDownService = "can't gracefully shut down service, force stop"
|
||||||
|
CantShutDownService = "can't shut down service"
|
||||||
|
SIGHUPConfigReloadStarted = "SIGHUP config reload started"
|
||||||
|
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed"
|
||||||
|
FailedToReloadConfig = "failed to reload config"
|
||||||
|
LogLevelWontBeUpdated = "log level won't be updated"
|
||||||
|
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed"
|
||||||
|
)
|
99
internal/metrics/desc.go
Normal file
99
internal/metrics/desc.go
Normal file
|
@ -0,0 +1,99 @@
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
dto "github.com/prometheus/client_model/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
var appMetricsDesc = map[string]map[string]Description{
|
||||||
|
stateSubsystem: {
|
||||||
|
healthMetric: Description{
|
||||||
|
Type: dto.MetricType_GAUGE,
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: stateSubsystem,
|
||||||
|
Name: healthMetric,
|
||||||
|
Help: "FrostFS S3 Lifecycler state",
|
||||||
|
},
|
||||||
|
versionInfoMetric: Description{
|
||||||
|
Type: dto.MetricType_GAUGE,
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: stateSubsystem,
|
||||||
|
Name: versionInfoMetric,
|
||||||
|
Help: "Version of current FrostFS S3 Lifecycler instance",
|
||||||
|
VariableLabels: []string{"version"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
type Description struct {
|
||||||
|
Type dto.MetricType
|
||||||
|
Namespace string
|
||||||
|
Subsystem string
|
||||||
|
Name string
|
||||||
|
Help string
|
||||||
|
ConstantLabels prometheus.Labels
|
||||||
|
VariableLabels []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Description) MarshalJSON() ([]byte, error) {
|
||||||
|
return json.Marshal(&struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
FQName string `json:"name"`
|
||||||
|
Help string `json:"help"`
|
||||||
|
ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"`
|
||||||
|
VariableLabels []string `json:"variable_labels,omitempty"`
|
||||||
|
}{
|
||||||
|
Type: d.Type.String(),
|
||||||
|
FQName: d.BuildFQName(),
|
||||||
|
Help: d.Help,
|
||||||
|
ConstantLabels: d.ConstantLabels,
|
||||||
|
VariableLabels: d.VariableLabels,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Description) BuildFQName() string {
|
||||||
|
return prometheus.BuildFQName(d.Namespace, d.Subsystem, d.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DescribeAll returns descriptions for metrics.
|
||||||
|
func DescribeAll() []Description {
|
||||||
|
var list []Description
|
||||||
|
for _, m := range appMetricsDesc {
|
||||||
|
for _, description := range m {
|
||||||
|
list = append(list, description)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return list
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOpts(description Description) prometheus.Opts {
|
||||||
|
return prometheus.Opts{
|
||||||
|
Namespace: description.Namespace,
|
||||||
|
Subsystem: description.Subsystem,
|
||||||
|
Name: description.Name,
|
||||||
|
Help: description.Help,
|
||||||
|
ConstLabels: description.ConstantLabels,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustNewGauge(description Description) prometheus.Gauge {
|
||||||
|
if description.Type != dto.MetricType_GAUGE {
|
||||||
|
panic("invalid metric type")
|
||||||
|
}
|
||||||
|
return prometheus.NewGauge(
|
||||||
|
prometheus.GaugeOpts(newOpts(description)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustNewGaugeVec(description Description) *prometheus.GaugeVec {
|
||||||
|
if description.Type != dto.MetricType_GAUGE {
|
||||||
|
panic("invalid metric type")
|
||||||
|
}
|
||||||
|
return prometheus.NewGaugeVec(
|
||||||
|
prometheus.GaugeOpts(newOpts(description)),
|
||||||
|
description.VariableLabels,
|
||||||
|
)
|
||||||
|
}
|
27
internal/metrics/desc_test.go
Normal file
27
internal/metrics/desc_test.go
Normal file
|
@ -0,0 +1,27 @@
|
||||||
|
//go:build dump_metrics
|
||||||
|
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
var metricsPath = flag.String("out", "", "File to export Frostfs S3 lifecycler metrics to.")
|
||||||
|
|
||||||
|
func TestDescribeAll(t *testing.T) {
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
require.NotEmpty(t, metricsPath, "flag 'out' must be provided to dump metrics description")
|
||||||
|
|
||||||
|
desc := DescribeAll()
|
||||||
|
data, err := json.Marshal(desc)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = os.WriteFile(*metricsPath, data, 0644)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
76
internal/metrics/metrics.go
Normal file
76
internal/metrics/metrics.go
Normal file
|
@ -0,0 +1,76 @@
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
// AppMetrics is a metrics container for all app specific data.
|
||||||
|
AppMetrics struct {
|
||||||
|
stateMetrics
|
||||||
|
}
|
||||||
|
|
||||||
|
// stateMetrics are metrics of application state.
|
||||||
|
stateMetrics struct {
|
||||||
|
healthCheck prometheus.Gauge
|
||||||
|
versionInfo *prometheus.GaugeVec
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
namespace = "frostfs_s3_lifecycler"
|
||||||
|
stateSubsystem = "state"
|
||||||
|
|
||||||
|
healthMetric = "health"
|
||||||
|
versionInfoMetric = "version_info"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (m stateMetrics) register() {
|
||||||
|
prometheus.MustRegister(m.healthCheck)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m stateMetrics) SetHealth(s int32) {
|
||||||
|
m.healthCheck.Set(float64(s))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m stateMetrics) SetVersion(ver string) {
|
||||||
|
m.versionInfo.WithLabelValues(ver).Set(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAppMetrics creates an instance of application.
|
||||||
|
func NewAppMetrics() *AppMetrics {
|
||||||
|
stateMetric := newStateMetrics()
|
||||||
|
stateMetric.register()
|
||||||
|
|
||||||
|
return &AppMetrics{
|
||||||
|
stateMetrics: *stateMetric,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStateMetrics() *stateMetrics {
|
||||||
|
return &stateMetrics{
|
||||||
|
healthCheck: mustNewGauge(appMetricsDesc[stateSubsystem][healthMetric]),
|
||||||
|
versionInfo: mustNewGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPrometheusService creates a new service for gathering prometheus metrics.
|
||||||
|
func NewPrometheusService(log *zap.Logger, cfg Config) *Service {
|
||||||
|
if log == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Service{
|
||||||
|
Server: &http.Server{
|
||||||
|
Addr: cfg.Address,
|
||||||
|
Handler: promhttp.Handler(),
|
||||||
|
},
|
||||||
|
enabled: cfg.Enabled,
|
||||||
|
serviceType: "Prometheus",
|
||||||
|
log: log.With(zap.String("service", "Prometheus")),
|
||||||
|
}
|
||||||
|
}
|
33
internal/metrics/profiler.go
Normal file
33
internal/metrics/profiler.go
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/pprof"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewPprofService creates a new service for gathering pprof metrics.
|
||||||
|
func NewPprofService(l *zap.Logger, cfg Config) *Service {
|
||||||
|
handler := http.NewServeMux()
|
||||||
|
handler.HandleFunc("/debug/pprof/", pprof.Index)
|
||||||
|
handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
|
||||||
|
handler.HandleFunc("/debug/pprof/profile", pprof.Profile)
|
||||||
|
handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
|
||||||
|
handler.HandleFunc("/debug/pprof/trace", pprof.Trace)
|
||||||
|
|
||||||
|
// Manually add support for paths linked to by index page at /debug/pprof/
|
||||||
|
for _, item := range []string{"allocs", "block", "heap", "goroutine", "mutex", "threadcreate"} {
|
||||||
|
handler.Handle("/debug/pprof/"+item, pprof.Handler(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Service{
|
||||||
|
Server: &http.Server{
|
||||||
|
Addr: cfg.Address,
|
||||||
|
Handler: handler,
|
||||||
|
},
|
||||||
|
enabled: cfg.Enabled,
|
||||||
|
serviceType: "Pprof",
|
||||||
|
log: l.With(zap.String("service", "Pprof")),
|
||||||
|
}
|
||||||
|
}
|
49
internal/metrics/service.go
Normal file
49
internal/metrics/service.go
Normal file
|
@ -0,0 +1,49 @@
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Service serves metrics.
|
||||||
|
type Service struct {
|
||||||
|
*http.Server
|
||||||
|
enabled bool
|
||||||
|
log *zap.Logger
|
||||||
|
serviceType string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config is a params to configure service.
|
||||||
|
type Config struct {
|
||||||
|
Address string
|
||||||
|
Enabled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start runs http service with the exposed endpoint on the configured port.
|
||||||
|
func (ms *Service) Start() {
|
||||||
|
if ms.enabled {
|
||||||
|
// nolint: truecloudlab-linters
|
||||||
|
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr))
|
||||||
|
err := ms.ListenAndServe()
|
||||||
|
if err != nil && err != http.ErrServerClosed {
|
||||||
|
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShutDown stops the service.
|
||||||
|
func (ms *Service) ShutDown(ctx context.Context) {
|
||||||
|
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr))
|
||||||
|
err := ms.Shutdown(ctx)
|
||||||
|
if err != nil {
|
||||||
|
ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err))
|
||||||
|
if err = ms.Close(); err != nil {
|
||||||
|
ms.log.Panic(logs.CantShutDownService, zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue